curl: add --no-clobber

Does not overwrite output files if they already exist

Closes #7708
Co-authored-by: Daniel Stenberg

docs/TODO

@@ -142,7 +142,6 @@
  18. Command line tool
  18.1 sync
  18.2 glob posts
- 18.3 prevent file overwriting
  18.4 --proxycommand
  18.5 UTF-8 filenames in Content-Disposition
  18.6 Option to make -Z merge lined based outputs on stdout
@@ -940,14 +939,6 @@
  Globbing support for -d and -F, as in 'curl -d "name=foo[0-9]" URL'.
  This is easily scripted though.
 
-18.3 prevent file overwriting
-
- Add an option that prevents curl from overwriting existing local files. When
- used, and there already is an existing file with the target file name
- (either -O or -o), a number should be appended (and increased if already
- existing). So that index.html becomes first index.html.1 and then
- index.html.2 etc.
-
 18.4 --proxycommand
 
  Allow the user to make curl run a command and use its stdio to make requests

docs/cmdline-opts/Makefile.inc

@@ -140,6 +140,7 @@ DPAGES = \
   next.d \
   no-alpn.d \
   no-buffer.d \
+  no-clobber.d \
   no-keepalive.d \
   no-npn.d \
   no-progress-meter.d \

docs/cmdline-opts/no-clobber.d

@@ -0,0 +1,16 @@
+Long: no-clobber
+Help: Do not overwrite files that already exist
+Category: curl output
+Added: 7.83.0
+See-also: output remote-name
+Example: --no-clobber --output local/dir/file $URL
+---
+When used in conjunction with the --output, --remote-header-name,
+--remote-name, or --remote-name-all options, curl avoids overwriting files
+that already exist. Instead, a dot and a number gets appended to the name
+of the file that would be created, up to filename.100 after which it will not
+create any file.
+
+Note that this is the negated option name documented.  You can thus use
+--clobber to enforce the clobbering, even if --remote-header-name or -J is
+specified.

docs/options-in-versions

@@ -128,6 +128,7 @@
 --next (-:)                          7.36.0
 --no-alpn                            7.36.0
 --no-buffer (-N)                     6.5
+--no-clobber                         7.83.0
 --no-keepalive                       7.18.0
 --no-npn                             7.36.0
 --no-progress-meter                  7.67.0

src/tool_cb_wrt.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -48,50 +48,86 @@
 #define OPENMODE S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH
 #endif
 
-/* create a local file for writing, return TRUE on success */
+/* create/open a local file for writing, return TRUE on success */
 bool tool_create_output_file(struct OutStruct *outs,
                              struct OperationConfig *config)
 {
   struct GlobalConfig *global;
   FILE *file = NULL;
+  char *fname = outs->filename;
+  char *aname = NULL;
   DEBUGASSERT(outs);
   DEBUGASSERT(config);
   global = config->global;
-  if(!outs->filename || !*outs->filename) {
+  if(!fname || !*fname) {
     warnf(global, "Remote filename has no length!\n");
     return FALSE;
   }
 
-  if(outs->is_cd_filename) {
+  if(config->output_dir && outs->is_cd_filename) {
-    /* don't overwrite existing files */
+    aname = aprintf("%s/%s", config->output_dir, fname);
-    int fd;
-    char *name = outs->filename;
-    char *aname = NULL;
-    if(config->output_dir) {
-      aname = aprintf("%s/%s", config->output_dir, name);
     if(!aname) {
       errorf(global, "out of memory\n");
       return FALSE;
     }
-      name = aname;
+    fname = aname;
   }
-    fd = open(name, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE);
+
+  if(config->file_clobber_mode == CLOBBER_ALWAYS ||
+     (config->file_clobber_mode == CLOBBER_DEFAULT &&
+      !outs->is_cd_filename)) {
+    /* open file for writing */
+    file = fopen(fname, "wb");
+  }
+  else {
+    int fd;
+    do {
+      fd = open(fname, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE);
+      /* Keep retrying in the hope that it isn't interrupted sometime */
+    } while(fd == -1 && errno == EINTR);
+    if(config->file_clobber_mode == CLOBBER_NEVER && fd == -1) {
+      int next_num = 1;
+      size_t len = strlen(fname);
+      char *newname = malloc(len + 13); /* nul + 1-11 digits + dot */
+      if(!newname) {
+        errorf(global, "out of memory\n");
+        return FALSE;
+      }
+      memcpy(newname, fname, len);
+      newname[len] = '.';
+      while(fd == -1 && /* haven't sucessfully opened a file */
+            (errno == EEXIST || errno == EISDIR) &&
+            /* because we keep having files that already exist */
+            next_num < 100 /* and we haven't reached the retry limit */ ) {
+        curlx_msnprintf(newname + len + 1, 12, "%d", next_num);
+        next_num++;
+        do {
+          fd = open(newname, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE);
+          /* Keep retrying in the hope that it isn't interrupted sometime */
+        } while(fd == -1 && errno == EINTR);
+      }
+      outs->filename = newname; /* remember the new one */
+      outs->alloc_filename = TRUE;
+    }
+    /* An else statement to not overwrite existing files and not retry with
+       new numbered names (which would cover
+       config->file_clobber_mode == CLOBBER_DEFAULT && outs->is_cd_filename)
+       is not needed because we would have failed earlier, in the while loop
+       and `fd` would now be -1 */
     if(fd != -1) {
       file = fdopen(fd, "wb");
       if(!file)
         close(fd);
     }
-    free(aname);
   }
-  else
-    /* open file for writing */
-    file = fopen(outs->filename, "wb");
 
   if(!file) {
-    warnf(global, "Failed to create the file %s: %s\n", outs->filename,
+    warnf(global, "Failed to open the file %s: %s\n", fname,
           strerror(errno));
+    free(aname);
     return FALSE;
   }
+  free(aname);
   outs->s_isreg = TRUE;
   outs->fopened = TRUE;
   outs->stream = file;

src/tool_cfgable.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -45,6 +45,7 @@ void config_init(struct OperationConfig *config)
   config->happy_eyeballs_timeout_ms = CURL_HET_DEFAULT;
   config->http09_allowed = FALSE;
   config->ftp_skip_ip = TRUE;
+  config->file_clobber_mode = CLOBBER_DEFAULT;
 }
 
 static void free_config_fields(struct OperationConfig *config)

src/tool_cfgable.h

@@ -290,6 +290,15 @@ struct OperationConfig {
   bool haproxy_protocol;          /* whether to send HAProxy protocol v1 */
   bool disallow_username_in_url;  /* disallow usernames in URLs */
   char *aws_sigv4;
+  enum {
+    CLOBBER_DEFAULT, /* Provides compatability with previous versions of curl,
+                        by using the default behavior for -o, -O, and -J.
+                        If those options would have overwritten files, like
+                        -o and -O would, then overwrite them. In the case of
+                        -J, this will not overwrite any files. */
+    CLOBBER_NEVER, /* If the file exists, always fail */
+    CLOBBER_ALWAYS /* If the file exists, always overwrite it */
+  } file_clobber_mode;
   struct GlobalConfig *global;
   struct OperationConfig *prev;
   struct OperationConfig *next;   /* Always last in the struct */

src/tool_getparam.c

@@ -314,6 +314,7 @@ static const struct LongShort aliases[]= {
   {"O",  "remote-name",              ARG_NONE},
   {"Oa", "remote-name-all",          ARG_BOOL},
   {"Ob", "output-dir",               ARG_STRING},
+  {"Oc", "clobber",                  ARG_BOOL},
   {"p",  "proxytunnel",              ARG_BOOL},
   {"P",  "ftp-port",                 ARG_STRING},
   {"q",  "disable",                  ARG_BOOL},
@@ -1999,10 +2000,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
     case 'N':
       /* disable the output I/O buffering. note that the option is called
          --buffer but is mostly used in the negative form: --no-buffer */
-      if(longopt)
+      config->nobuffer = longopt ? !toggle : TRUE;
-        config->nobuffer = (!toggle)?TRUE:FALSE;
-      else
-        config->nobuffer = toggle;
       break;
     case 'O': /* --remote-name */
       if(subletter == 'a') { /* --remote-name-all */
@@ -2013,6 +2011,10 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
         GetStr(&config->output_dir, nextarg);
         break;
       }
+      else if(subletter == 'c') { /* --clobber / --no-clobber */
+        config->file_clobber_mode = toggle ? CLOBBER_ALWAYS : CLOBBER_NEVER;
+        break;
+      }
       /* FALLTHROUGH */
     case 'o': /* --output */
       /* output file */

src/tool_help.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms

src/tool_listhelp.c

@@ -385,6 +385,9 @@ const struct helptxt helptext[] = {
   {"-N, --no-buffer",
    "Disable buffering of the output stream",
    CURLHELP_CURL},
+  {"    --no-clobber",
+   "Do not overwrite files that already exist",
+   CURLHELP_CURL | CURLHELP_OUTPUT},
   {"    --no-keepalive",
    "Disable TCP keepalive on the connection",
    CURLHELP_CONNECTION},

tests/data/Makefile.inc

@@ -209,6 +209,8 @@ test1630 test1631 test1632 test1633 test1634 \
 test1650 test1651 test1652 test1653 test1654 test1655 \
 test1660 test1661 \
 \
+test1680 test1681 test1682 test1683 \
+\
 test1700 test1701 test1702 test1703 \
 \
 test1800 test1801 \

tests/data/test1680

@@ -0,0 +1,55 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+--clobber
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<name>
+HTTP GET with explicit clobber
+</name>
+<server>
+http
+</server>
+<features>
+http
+</features>
+<command option="no-output">
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --clobber
+</command>
+<file name="log/exist%TESTNUMBER">
+to be overwritten
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<file name="log/exist%TESTNUMBER">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</file>
+</verify>
+</testcase>

tests/data/test1681

@@ -0,0 +1,61 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+--no-clobber
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<name>
+HTTP GET without clobber
+</name>
+<server>
+http
+</server>
+<features>
+http
+</features>
+<command option="no-output">
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --no-clobber -w '%{filename_effective}\n'
+</command>
+<file name="log/exist%TESTNUMBER">
+to stay the same
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<file name="log/exist%TESTNUMBER">
+to stay the same
+</file>
+<file1 name="log/exist%TESTNUMBER.1">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</file1>
+<stdout mode="text">
+log/exist%TESTNUMBER.1
+</stdout>
+</verify>
+</testcase>

tests/data/test1682

@@ -0,0 +1,58 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+--no-clobber
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<name>
+HTTP GET without clobber and --output-dir
+</name>
+<server>
+http
+</server>
+<features>
+http
+</features>
+<command option="no-output">
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --output-dir log -o exist%TESTNUMBER --no-clobber
+</command>
+<file name="log/exist%TESTNUMBER">
+to stay the same
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<file name="log/exist%TESTNUMBER">
+to stay the same
+</file>
+<file1 name="log/exist%TESTNUMBER.1">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</file1>
+</verify>
+</testcase>

tests/data/test1683

@@ -0,0 +1,61 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+--no-clobber
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.0 200 OK
+Connection: close
+Content-Type: text/plain
+Content-Length: 4
+
+foo
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<name>
+HTTP GET without clobber when 100 files already exist
+</name>
+<server>
+http
+</server>
+<features>
+http
+</features>
+<command option="no-output">
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --no-clobber
+</command>
+<file name="log/exist%TESTNUMBER">
+to stay the same
+</file>
+<precheck>
+perl -e 'for my $i ((1..100)) { my $filename = "log/exist%TESTNUMBER.$i"; open(FH, ">", $filename) or die $!; print FH "to stay the same" ; close(FH) }'
+# python3 -c 'for i in range(1, 101): open("log/exist%TESTNUMBER.{}".format(i), mode="w").write("to stay the same")'
+</precheck>
+<postcheck>
+perl -e 'for my $i ((1..100)) { my $filename = "log/exist%TESTNUMBER.$i"; open(FH, "<", $filename) or die $!; (<FH> eq "to stay the same" and <FH> eq "") or die "incorrect $filename" ; close(FH) }'
+# python3 -c 'for i in range(1, 101): assert open("log/exist%TESTNUMBER.{}".format(i), mode="r").read(17) == "to stay the same"'
+</postcheck>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+23
+</errorcode>
+<file name="log/exist%TESTNUMBER">
+to stay the same
+</file>
+</verify>
+</testcase>

tests/manpage-scan.pl

@@ -6,7 +6,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 2016 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 2016 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -146,6 +146,7 @@ my %opts = (
     '--no-sessionid' => 1,
     '--no-keepalive' => 1,
     '--no-progress-meter' => 1,
+    '--no-clobber' => 1,
 
     # pretend these options without -no exist in curl.1 and tool_listhelp.c
     '--alpn' => 6,
@@ -156,8 +157,9 @@ my %opts = (
     '-N, --buffer' => 6,
     '--sessionid' => 6,
     '--progress-meter' => 6,
+    '--clobber' => 6,
 
-    # deprecated options do not need to be in tool_listhelp.c nor curl.1
+    # deprecated options do not need to be in tool_help.c nor curl.1
     '--krb4' => 6,
     '--ftp-ssl' => 6,
     '--ftp-ssl-reqd' => 6,