curl: add --skip-existing

With this option, the entire download is skipped if the selected target
filename already exists when the opertion is about to begin.

Test 994, 995 and 996 verify.

Ref: #11012
Closes #13993

docs/cmdline-opts/Makefile.inc

@@ -253,6 +253,7 @@ DPAGES = \
   show-error.md \
   show-headers.md \
   silent.md \
+  skip-existing.md \
   socks4.md \
   socks4a.md \
   socks5-basic.md \

docs/cmdline-opts/skip-existing.md

@@ -0,0 +1,22 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Long: skip-existing
+Help: Skip download if local file already exists
+Category: curl output
+Added: 8.10.0
+Multi: boolean
+See-also:
+  - output
+  - remote-name
+  - no-clobber
+Example:
+  - --skip-existing --output local/dir/file $URL
+---
+
+# `--skip-existing`
+
+If there is a local file present when a download is requested, the operation
+is skipped. Note that curl cannot know if the local file was previously
+downloaded fine, or if it is incomplete etc, it just knows if there is a
+filename present in the file system or not and it skips the transfer if it is.

docs/options-in-versions

@@ -218,6 +218,7 @@
 --show-error (-S)                    5.9
 --show-headers (-i)                  4.8
 --silent (-s)                        4.0
+--skip-existing                      8.10.0
 --socks4                             7.15.2
 --socks4a                            7.18.0
 --socks5                             7.18.0

src/tool_cfgable.h

@@ -302,6 +302,7 @@ struct OperationConfig {
   struct State state;             /* for create_transfer() */
   bool rm_partial;                /* on error, remove partially written output
                                      files */
+  bool skip_existing;
 #ifdef USE_ECH
   char *ech;                      /* Config set by --ech keywords */
   char *ech_config;               /* Config set by "--ech esl:" option */

src/tool_getparam.c

@@ -287,6 +287,7 @@ static const struct LongShort aliases[]= {
   {"show-error",                 ARG_BOOL, 'S', C_SHOW_ERROR},
   {"show-headers",               ARG_BOOL, 'i', C_SHOW_HEADERS},
   {"silent",                     ARG_BOOL, 's', C_SILENT},
+  {"skip-existing",              ARG_BOOL, ' ', C_SKIP_EXISTING},
   {"socks4",                     ARG_STRG, ' ', C_SOCKS4},
   {"socks4a",                    ARG_STRG, ' ', C_SOCKS4A},
   {"socks5",                     ARG_STRG, ' ', C_SOCKS5},
@@ -2435,6 +2436,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
     case C_SILENT: /* --silent */
       global->silent = toggle;
       break;
+    case C_SKIP_EXISTING: /* --skip-existing */
+      config->skip_existing = toggle;
+      break;
     case C_SHOW_ERROR: /* --show-error */
       global->showerror = toggle;
       break;

src/tool_getparam.h

@@ -242,6 +242,7 @@ typedef enum {
   C_SHOW_ERROR,
   C_SHOW_HEADERS,
   C_SILENT,
+  C_SKIP_EXISTING,
   C_SOCKS4,
   C_SOCKS4A,
   C_SOCKS5,

src/tool_listhelp.c

@@ -662,6 +662,9 @@ const struct helptxt helptext[] = {
   {"-s, --silent",
    "Silent mode",
    CURLHELP_IMPORTANT | CURLHELP_VERBOSE},
+  {"    --skip-existing",
+   "Skip download if local file already exists",
+   CURLHELP_CURL | CURLHELP_OUTPUT},
   {"    --socks4 <host[:port]>",
    "SOCKS4 proxy on given host + port",
    CURLHELP_PROXY},

src/tool_operate.c

@@ -460,6 +460,9 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
   if(per->infdopen)
     close(per->infd);
 
+  if(per->skip)
+    goto skip;
+
 #ifdef __VMS
   if(is_vms_shell()) {
     /* VMS DCL shell behavior */
@@ -731,7 +734,7 @@ noretry:
     curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime);
     setfiletime(filetime, outs->filename, global);
   }
-
+skip:
   /* Write the --write-out data before cleanup but after result is final */
   if(config->writeout)
     ourWriteOut(config, per, result);
@@ -1197,6 +1200,15 @@ static CURLcode single_transfer(struct GlobalConfig *global,
               break;
           }
 
+          if(per->outfile && config->skip_existing) {
+            struct_stat fileinfo;
+            if(!stat(per->outfile, &fileinfo)) {
+              /* file is present */
+              notef(global, "skips transfer, \"%s\" exists locally",
+                    per->outfile);
+              per->skip = TRUE;
+            }
+          }
           if((urlnode->flags & GETOUT_USEREMOTE)
              && config->content_disposition) {
             /* Our header callback MIGHT set the filename */
@@ -2611,6 +2623,9 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
     long delay_ms;
     bool bailout = FALSE;
     struct timeval start;
+
+    start = tvnow();
+    if(!per->skip) {
       result = pre_transfer(global, per);
       if(result)
         break;
@@ -2620,7 +2635,7 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
         if(result)
           break;
       }
-    start = tvnow();
+
 #ifdef DEBUGBUILD
       if(getenv("CURL_FORBID_REUSE"))
         (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
@@ -2630,6 +2645,7 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
       else
 #endif
         result = curl_easy_perform(per->curl);
+    }
 
     returncode = post_per_transfer(global, per, result, &retry, &delay_ms);
     if(retry) {

src/tool_operate.h

@@ -44,25 +44,16 @@ struct per_transfer {
   char *this_url;
   unsigned int urlnum; /* the index of the given URL */
   char *outfile;
-  bool infdopen; /* TRUE if infd needs closing */
   int infd;
-  bool noprogress;
   struct ProgressData progressbar;
   struct OutStruct outs;
   struct OutStruct heads;
   struct OutStruct etag_save;
   struct HdrCbData hdrcbdata;
   long num_headers;
-  bool was_last_header_empty;
-
-  bool added; /* set TRUE when added to the multi handle */
   time_t startat; /* when doing parallel transfers, this is a retry transfer
                      that has been set to sleep until this time before it
                      should get started (again) */
-  bool abort; /* when doing parallel transfers and this is TRUE then a critical
-                 error (eg --fail-early) has occurred in another transfer and
-                 this transfer will be aborted in the progress callback */
-
   /* for parallel progress bar */
   curl_off_t dltotal;
   curl_off_t dlnow;
@@ -77,6 +68,15 @@ struct per_transfer {
   char *uploadfile;
   char *errorbuffer; /* allocated and assigned while this is used for a
                         transfer */
+  bool infdopen; /* TRUE if infd needs closing */
+  bool noprogress;
+  bool was_last_header_empty;
+
+  bool added; /* set TRUE when added to the multi handle */
+  bool abort; /* when doing parallel transfers and this is TRUE then a critical
+                 error (eg --fail-early) has occurred in another transfer and
+                 this transfer will be aborted in the progress callback */
+  bool skip;  /* considered already done */
 };
 
 CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]);

tests/data/Makefile.am

@@ -129,7 +129,7 @@ test952 test953 test954 test955 test956 test957 test958 test959 test960 \
 test961 test962 test963 test964 test965 test966 test967 test968 test969 \
 test970 test971 test972 test973 test974 test975 test976 test977 test978 \
 test979 test980 test981 test982 test983 test984 test985 test986 test987 \
-test988 test989 test990 test991 test992 test993 \
+test988 test989 test990 test991 test992 test993 test994 test995 test996 \
 \
 test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 \
 test1008 test1009 test1010 test1011 test1012 test1013 test1014 test1015 \

tests/data/test994

@@ -0,0 +1,42 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--skip-existing with globbing
+</name>
+<command option="no-output">
+-o "%LOGDIR/#1" "http://%HOSTIP:%HTTPPORT/%TESTNUMBER/{hey,ho}" --skip-existing
+</command>
+<file name="%LOGDIR/hey">
+content
+</file>
+<file2 name="%LOGDIR/ho">
+content
+</file2>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stderr mode="text">
+Note: skips transfer, "%LOGDIR/hey" exists locally
+Note: skips transfer, "%LOGDIR/ho" exists locally
+</stderr>
+</verify>
+</testcase>

tests/data/test995

@@ -0,0 +1,56 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--skip-existing without file present
+</name>
+<command option="no-output,no-include">
+-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+<file name="%LOGDIR/there">
+-foo-
+</file>
+</verify>
+</testcase>

tests/data/test996

@@ -0,0 +1,41 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--skip-existing with file present
+</name>
+<command option="no-output">
+-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing
+</command>
+<file name="%LOGDIR/there">
+content
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stderr mode="text">
+Note: skips transfer, "%LOGDIR/there" exists locally
+</stderr>
+<file name="%LOGDIR/there">
+content
+</file>
+</verify>
+</testcase>