file: add support for getting basic directory listings

Not supported on Windows (yet)

Closes #13137

CMake/Platforms/WindowsCache.cmake

@@ -137,6 +137,9 @@ set(HAVE_TERMIOS_H 0)
 set(HAVE_TERMIO_H 0)
 set(HAVE_UTIME_H 0)  # mingw-w64 has it (wrapper to sys/utime.h)
 
+set(HAVE_DIRENT_H 0)
+set(HAVE_OPENDIR 0)
+
 set(HAVE_FSEEKO 0)
 set(HAVE__FSEEKI64 1)
 set(HAVE_SOCKET 1)

CMakeLists.txt

@@ -1126,6 +1126,7 @@ check_include_file_concat("sys/un.h"         HAVE_SYS_UN_H)
 check_include_file_concat("sys/utime.h"      HAVE_SYS_UTIME_H)
 check_include_file_concat("sys/xattr.h"      HAVE_SYS_XATTR_H)
 check_include_file_concat("arpa/inet.h"      HAVE_ARPA_INET_H)
+check_include_file_concat("dirent.h"         HAVE_DIRENT_H)
 check_include_file_concat("fcntl.h"          HAVE_FCNTL_H)
 check_include_file_concat("ifaddrs.h"        HAVE_IFADDRS_H)
 check_include_file_concat("io.h"             HAVE_IO_H)
@@ -1188,6 +1189,7 @@ endif()
 
 check_symbol_exists(fnmatch       "${CURL_INCLUDES};fnmatch.h" HAVE_FNMATCH)
 check_symbol_exists(basename      "${CURL_INCLUDES};string.h" HAVE_BASENAME)
+check_symbol_exists(opendir       "${CURL_INCLUDES};dirent.h" HAVE_OPENDIR)
 check_symbol_exists(socket        "${CURL_INCLUDES}" HAVE_SOCKET)
 check_symbol_exists(sched_yield   "${CURL_INCLUDES};sched.h" HAVE_SCHED_YIELD)
 check_symbol_exists(socketpair    "${CURL_INCLUDES}" HAVE_SOCKETPAIR)

configure.ac

@@ -3984,6 +3984,12 @@ if test "$want_thres" = "yes" && test "x$USE_THREADS_POSIX" != "x1"; then
   fi
 fi
 
+AC_CHECK_HEADER(dirent.h,
+  [ AC_DEFINE(HAVE_DIRENT_H, 1, [if you have <dirent.h>])
+    AC_CHECK_FUNC(opendir, AC_DEFINE(HAVE_OPENDIR, 1, [if you have opendir]) )
+  ]
+)
+
 CURL_CONVERT_INCLUDE_TO_ISYSTEM
 
 dnl ************************************************************

docs/cmdline-opts/list-only.md

@@ -3,10 +3,10 @@ c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
 SPDX-License-Identifier: curl
 Long: list-only
 Short: l
-Protocols: FTP POP3 SFTP
+Protocols: FTP POP3 SFTP FILE
 Help: List only mode
 Added: 4.0
-Category: ftp pop3 sftp
+Category: ftp pop3 sftp file
 Multi: boolean
 See-also:
   - quote
@@ -35,6 +35,9 @@ When retrieving a specific email from POP3, this switch forces a LIST command
 to be performed instead of RETR. This is particularly useful if the user wants
 to see if a specific message-id exists on the server and what size it is.
 
+For FILE, this option has no effect yet as directories are always listed in
+this mode.
+
 Note: When combined with --request, this option can be used to send a UIDL
 command instead, so the user may use the email's unique identifier rather than
 its message-id to make the request.

docs/libcurl/opts/CURLOPT_DIRLISTONLY.md

@@ -36,6 +36,9 @@ messages on the POP3 server. This can be used to change the default behavior
 of libcurl, when combined with a URL that contains a message ID, to perform a
 "scan listing" which can then be used to determine the size of an email.
 
+For FILE, this option has no effect yet as directories are always listed in
+this mode.
+
 Note: For FTP this causes a NLST command to be sent to the FTP server. Beware
 that some FTP servers list only files in their response to NLST; they might
 not include subdirectories and symbolic links.

lib/curl_config.h.cmake

@@ -199,6 +199,12 @@
 /* Define to 1 if you have the `closesocket' function. */
 #cmakedefine HAVE_CLOSESOCKET 1
 
+/* Define to 1 if you have the <dirent.h> header file. */
+#cmakedefine HAVE_DIRENT_H 1
+
+/* Define to 1 if you have the `opendir' function. */
+#cmakedefine HAVE_OPENDIR 1
+
 /* Define to 1 if you have the fcntl function. */
 #cmakedefine HAVE_FCNTL 1
 

lib/file.c

@@ -50,6 +50,14 @@
 #include <fcntl.h>
 #endif
 
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+
 #include "strtoofft.h"
 #include "urldata.h"
 #include <curl/curl.h>
@@ -544,49 +552,85 @@ static CURLcode file_do(struct Curl_easy *data, bool *done)
     Curl_pgrsSetDownloadSize(data, expected_size);
 
   if(data->state.resume_from) {
-    if(data->state.resume_from !=
+    if(!S_ISDIR(statbuf.st_mode)) {
-       lseek(fd, data->state.resume_from, SEEK_SET))
+      if(data->state.resume_from !=
+          lseek(fd, data->state.resume_from, SEEK_SET))
+        return CURLE_BAD_DOWNLOAD_RESUME;
+    }
+    else {
       return CURLE_BAD_DOWNLOAD_RESUME;
+    }
   }
 
   result = Curl_multi_xfer_buf_borrow(data, &xfer_buf, &xfer_blen);
   if(result)
     goto out;
 
-  while(!result) {
+  if(!S_ISDIR(statbuf.st_mode)) {
-    ssize_t nread;
+    while(!result) {
-    /* Don't fill a whole buffer if we want less than all data */
+      ssize_t nread;
-    size_t bytestoread;
+      /* Don't fill a whole buffer if we want less than all data */
+      size_t bytestoread;
 
-    if(size_known) {
+      if(size_known) {
-      bytestoread = (expected_size < (curl_off_t)(xfer_blen-1)) ?
+        bytestoread = (expected_size < (curl_off_t)(xfer_blen-1)) ?
-        curlx_sotouz(expected_size) : (xfer_blen-1);
+          curlx_sotouz(expected_size) : (xfer_blen-1);
+      }
+      else
+        bytestoread = xfer_blen-1;
+
+      nread = read(fd, xfer_buf, bytestoread);
+
+      if(nread > 0)
+        xfer_buf[nread] = 0;
+
+      if(nread <= 0 || (size_known && (expected_size == 0)))
+        break;
+
+      if(size_known)
+        expected_size -= nread;
+
+      result = Curl_client_write(data, CLIENTWRITE_BODY, xfer_buf, nread);
+      if(result)
+        goto out;
+
+      if(Curl_pgrsUpdate(data))
+        result = CURLE_ABORTED_BY_CALLBACK;
+      else
+        result = Curl_speedcheck(data, Curl_now());
+      if(result)
+        goto out;
     }
-    else
-      bytestoread = xfer_blen-1;
-
-    nread = read(fd, xfer_buf, bytestoread);
-
-    if(nread > 0)
-      xfer_buf[nread] = 0;
-
-    if(nread <= 0 || (size_known && (expected_size == 0)))
-      break;
-
-    if(size_known)
-      expected_size -= nread;
-
-    result = Curl_client_write(data, CLIENTWRITE_BODY, xfer_buf, nread);
-    if(result)
-      goto out;
-
-    if(Curl_pgrsUpdate(data))
-      result = CURLE_ABORTED_BY_CALLBACK;
-    else
-      result = Curl_speedcheck(data, Curl_now());
-    if(result)
-      goto out;
   }
+  else {
+#ifdef HAVE_OPENDIR
+    DIR *dir = opendir(file->path);
+    struct dirent *entry;
+
+    if(!dir) {
+      result = CURLE_READ_ERROR;
+      goto out;
+    }
+    else {
+      while((entry = readdir(dir))) {
+        if(entry->d_name[0] != '.') {
+          result = Curl_client_write(data, CLIENTWRITE_BODY,
+                   entry->d_name, strlen(entry->d_name));
+          if(result)
+            break;
+          result = Curl_client_write(data, CLIENTWRITE_BODY, "\n", 1);
+          if(result)
+            break;
+        }
+      }
+      closedir(dir);
+    }
+#else
+    failf(data, "Directory listing not yet implemented on this platform.");
+    result = CURLE_READ_ERROR;
+#endif
+  }
+
   if(Curl_pgrsUpdate(data))
     result = CURLE_ABORTED_BY_CALLBACK;
 

lib/urldata.h

@@ -104,7 +104,7 @@ typedef unsigned int curl_prot_t;
 #define PROTO_FAMILY_SSH  (CURLPROTO_SCP|CURLPROTO_SFTP)
 
 #if !defined(CURL_DISABLE_FTP) || defined(USE_SSH) ||   \
-  !defined(CURL_DISABLE_POP3)
+  !defined(CURL_DISABLE_POP3) || !defined(CURL_DISABLE_FILE)
 /* these protocols support CURLOPT_DIRLISTONLY */
 #define CURL_LIST_ONLY_PROTOCOL 1
 #endif

src/tool_listhelp.c

@@ -338,7 +338,7 @@ const struct helptxt helptext[] = {
    CURLHELP_CONNECTION},
   {"-l, --list-only",
    "List only mode",
-   CURLHELP_FTP | CURLHELP_POP3 | CURLHELP_SFTP},
+   CURLHELP_FTP | CURLHELP_POP3 | CURLHELP_SFTP | CURLHELP_FILE},
   {"    --local-port <range>",
    "Use a local port number within RANGE",
    CURLHELP_CONNECTION},

tests/data/Makefile.inc

@@ -261,4 +261,4 @@ test3024 test3025 test3026 test3027 test3028 test3029 test3030 \
 \
 test3100 test3101 test3102 test3103 \
 test3200 \
-test3201 test3202
+test3201 test3202 test3203

tests/data/test1463

@@ -39,6 +39,7 @@ Usage: curl [options...] <url>
 file: FILE protocol options
      --create-file-mode <mode>  File mode for created files
  -I, --head                     Show document info only
+ -l, --list-only                List only mode
  -r, --range <range>            Retrieve only the bytes within RANGE
 </stdout>
 </verify>

tests/data/test1464

@@ -39,6 +39,7 @@ Usage: curl [options...] <url>
 file: FILE protocol options
      --create-file-mode <mode>  File mode for created files
  -I, --head                     Show document info only
+ -l, --list-only                List only mode
  -r, --range <range>            Retrieve only the bytes within RANGE
 </stdout>
 </verify>

tests/data/test3203

@@ -0,0 +1,41 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+FILE
+</keywords>
+</info>
+
+#
+# Client-side
+<client>
+<server>
+file
+</server>
+<name>
+GET a directory using file://
+</name>
+<!-- doesn't work on win32, see #6379 -->
+<features>
+!win32
+</features>
+<command option="no-include">
+file://localhost%FILE_PWD/%LOGDIR/test%TESTNUMBER.dir/
+</command>
+<file name="%LOGDIR/test%TESTNUMBER.dir/dir-listing-test.txt">
+Contents of file are irrelevant
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+0
+</errorcode>
+<stdout>
+dir-listing-test.txt
+</stdout>
+</verify>
+</testcase>