luo980/curl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

curl: add --no-clobber

Browse Source 
Does not overwrite output files if they already exist

Closes #7708
Co-authored-by: Daniel Stenberg
This commit is contained in:
HexTheDragon 2021-09-11 20:36:21 -08:00 committed by Daniel Stenberg
parent eed2e8e257
commit 1831a6e7f1
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
16 changed files with 333 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/TODO
View File

@ -142,7 +142,6 @@
18. Command line tool
18.1 sync
18.2 glob posts
18.3 prevent file overwriting
18.4 --proxycommand
18.5 UTF-8 filenames in Content-Disposition
18.6 Option to make -Z merge lined based outputs on stdout
@ -940,14 +939,6 @@
Globbing support for -d and -F, as in 'curl -d "name=foo[0-9]" URL'.
This is easily scripted though.
18.3 prevent file overwriting
Add an option that prevents curl from overwriting existing local files. When
used, and there already is an existing file with the target file name
(either -O or -o), a number should be appended (and increased if already
existing). So that index.html becomes first index.html.1 and then
index.html.2 etc.
18.4 --proxycommand
Allow the user to make curl run a command and use its stdio to make requests

1
docs/cmdline-opts/Makefile.inc
View File

@ -140,6 +140,7 @@ DPAGES = \
next.d \
no-alpn.d \
no-buffer.d \
no-clobber.d \
no-keepalive.d \
no-npn.d \
no-progress-meter.d \

16
docs/cmdline-opts/no-clobber.d Normal file
View File

@ -0,0 +1,16 @@
Long: no-clobber
Help: Do not overwrite files that already exist
Category: curl output
Added: 7.83.0
See-also: output remote-name
Example: --no-clobber --output local/dir/file $URL
---
When used in conjunction with the --output, --remote-header-name,
--remote-name, or --remote-name-all options, curl avoids overwriting files
that already exist. Instead, a dot and a number gets appended to the name
of the file that would be created, up to filename.100 after which it will not
create any file.
Note that this is the negated option name documented. You can thus use
--clobber to enforce the clobbering, even if --remote-header-name or -J is
specified.

1
docs/options-in-versions
View File

@ -128,6 +128,7 @@
--next (-:) 7.36.0
--no-alpn 7.36.0
--no-buffer (-N) 6.5
--no-clobber 7.83.0
--no-keepalive 7.18.0
--no-npn 7.36.0
--no-progress-meter 7.67.0

70
src/tool_cb_wrt.c
View File

@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@ -48,50 +48,86 @@
#define OPENMODE S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH
#endif
/* create a local file for writing, return TRUE on success */
/* create/open a local file for writing, return TRUE on success */
bool tool_create_output_file(struct OutStruct *outs,
struct OperationConfig *config)
{
struct GlobalConfig *global;
FILE *file = NULL;
char *fname = outs->filename;
char *aname = NULL;
DEBUGASSERT(outs);
DEBUGASSERT(config);
global = config->global;
if(!outs->filename || !*outs->filename) {
if(!fname || !*fname) {
warnf(global, "Remote filename has no length!\n");
return FALSE;
}
if(outs->is_cd_filename) {
/* don't overwrite existing files */
if(config->output_dir && outs->is_cd_filename) {
aname = aprintf("%s/%s", config->output_dir, fname);
if(!aname) {
errorf(global, "out of memory\n");
return FALSE;
}
fname = aname;
}
if(config->file_clobber_mode == CLOBBER_ALWAYS ||
(config->file_clobber_mode == CLOBBER_DEFAULT &&
!outs->is_cd_filename)) {
/* open file for writing */
file = fopen(fname, "wb");
}
else {
int fd;
char *name = outs->filename;
char *aname = NULL;
if(config->output_dir) {
aname = aprintf("%s/%s", config->output_dir, name);
if(!aname) {
do {
fd = open(fname, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE);
/* Keep retrying in the hope that it isn't interrupted sometime */
} while(fd == -1 && errno == EINTR);
if(config->file_clobber_mode == CLOBBER_NEVER && fd == -1) {
int next_num = 1;
size_t len = strlen(fname);
char *newname = malloc(len + 13); /* nul + 1-11 digits + dot */
if(!newname) {
errorf(global, "out of memory\n");
return FALSE;
}
name = aname;
memcpy(newname, fname, len);
newname[len] = '.';
while(fd == -1 && /* haven't sucessfully opened a file */
(errno == EEXIST || errno == EISDIR) &&
/* because we keep having files that already exist */
next_num < 100 /* and we haven't reached the retry limit */ ) {
curlx_msnprintf(newname + len + 1, 12, "%d", next_num);
next_num++;
do {
fd = open(newname, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE);
/* Keep retrying in the hope that it isn't interrupted sometime */
} while(fd == -1 && errno == EINTR);
}
outs->filename = newname; /* remember the new one */
outs->alloc_filename = TRUE;
}
fd = open(name, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE);
/* An else statement to not overwrite existing files and not retry with
new numbered names (which would cover
config->file_clobber_mode == CLOBBER_DEFAULT && outs->is_cd_filename)
is not needed because we would have failed earlier, in the while loop
and `fd` would now be -1 */
if(fd != -1) {
file = fdopen(fd, "wb");
if(!file)
close(fd);
}
free(aname);
}
else
/* open file for writing */
file = fopen(outs->filename, "wb");
if(!file) {
warnf(global, "Failed to create the file %s: %s\n", outs->filename,
warnf(global, "Failed to open the file %s: %s\n", fname,
strerror(errno));
free(aname);
return FALSE;
}
free(aname);
outs->s_isreg = TRUE;
outs->fopened = TRUE;
outs->stream = file;

3
src/tool_cfgable.c
View File

@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@ -45,6 +45,7 @@ void config_init(struct OperationConfig *config)
config->happy_eyeballs_timeout_ms = CURL_HET_DEFAULT;
config->http09_allowed = FALSE;
config->ftp_skip_ip = TRUE;
config->file_clobber_mode = CLOBBER_DEFAULT;
}
static void free_config_fields(struct OperationConfig *config)

9
src/tool_cfgable.h
View File

@ -290,6 +290,15 @@ struct OperationConfig {
bool haproxy_protocol; /* whether to send HAProxy protocol v1 */
bool disallow_username_in_url; /* disallow usernames in URLs */
char *aws_sigv4;
enum {
CLOBBER_DEFAULT, /* Provides compatability with previous versions of curl,
by using the default behavior for -o, -O, and -J.
If those options would have overwritten files, like
-o and -O would, then overwrite them. In the case of
-J, this will not overwrite any files. */
CLOBBER_NEVER, /* If the file exists, always fail */
CLOBBER_ALWAYS /* If the file exists, always overwrite it */
} file_clobber_mode;
struct GlobalConfig *global;
struct OperationConfig *prev;
struct OperationConfig *next; /* Always last in the struct */

10
src/tool_getparam.c
View File

@ -314,6 +314,7 @@ static const struct LongShort aliases[]= {
{"O", "remote-name", ARG_NONE},
{"Oa", "remote-name-all", ARG_BOOL},
{"Ob", "output-dir", ARG_STRING},
{"Oc", "clobber", ARG_BOOL},
{"p", "proxytunnel", ARG_BOOL},
{"P", "ftp-port", ARG_STRING},
{"q", "disable", ARG_BOOL},
@ -1999,10 +2000,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
case 'N':
/* disable the output I/O buffering. note that the option is called
--buffer but is mostly used in the negative form: --no-buffer */
if(longopt)
config->nobuffer = (!toggle)?TRUE:FALSE;
else
config->nobuffer = toggle;
config->nobuffer = longopt ? !toggle : TRUE;
break;
case 'O': /* --remote-name */
if(subletter == 'a') { /* --remote-name-all */
@ -2013,6 +2011,10 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
GetStr(&config->output_dir, nextarg);
break;
}
else if(subletter == 'c') { /* --clobber / --no-clobber */
config->file_clobber_mode = toggle ? CLOBBER_ALWAYS : CLOBBER_NEVER;
break;
}
/* FALLTHROUGH */
case 'o': /* --output */
/* output file */

2
src/tool_help.c
View File

@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms

3
src/tool_listhelp.c
View File

@ -385,6 +385,9 @@ const struct helptxt helptext[] = {
{"-N, --no-buffer",
"Disable buffering of the output stream",
CURLHELP_CURL},
{" --no-clobber",
"Do not overwrite files that already exist",
CURLHELP_CURL | CURLHELP_OUTPUT},
{" --no-keepalive",
"Disable TCP keepalive on the connection",
CURLHELP_CONNECTION},

2
tests/data/Makefile.inc
View File

@ -209,6 +209,8 @@ test1630 test1631 test1632 test1633 test1634 \
test1650 test1651 test1652 test1653 test1654 test1655 \
test1660 test1661 \
\
test1680 test1681 test1682 test1683 \
\
test1700 test1701 test1702 test1703 \
\
test1800 test1801 \

55
tests/data/test1680 Normal file
View File

@ -0,0 +1,55 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
--clobber
</keywords>
</info>
#
# Server-side
<reply>
<data nocheck="yes">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</data>
</reply>
#
# Client-side
<client>
<name>
HTTP GET with explicit clobber
</name>
<server>
http
</server>
<features>
http
</features>
<command option="no-output">
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --clobber
</command>
<file name="log/exist%TESTNUMBER">
to be overwritten
</file>
</client>
#
# Verify data after the test has been "shot"
<verify>
<file name="log/exist%TESTNUMBER">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</file>
</verify>
</testcase>

61
tests/data/test1681 Normal file
View File

@ -0,0 +1,61 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
--no-clobber
</keywords>
</info>
#
# Server-side
<reply>
<data nocheck="yes">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</data>
</reply>
#
# Client-side
<client>
<name>
HTTP GET without clobber
</name>
<server>
http
</server>
<features>
http
</features>
<command option="no-output">
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --no-clobber -w '%{filename_effective}\n'
</command>
<file name="log/exist%TESTNUMBER">
to stay the same
</file>
</client>
#
# Verify data after the test has been "shot"
<verify>
<file name="log/exist%TESTNUMBER">
to stay the same
</file>
<file1 name="log/exist%TESTNUMBER.1">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</file1>
<stdout mode="text">
log/exist%TESTNUMBER.1
</stdout>
</verify>
</testcase>

58
tests/data/test1682 Normal file
View File

@ -0,0 +1,58 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
--no-clobber
</keywords>
</info>
#
# Server-side
<reply>
<data nocheck="yes">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</data>
</reply>
#
# Client-side
<client>
<name>
HTTP GET without clobber and --output-dir
</name>
<server>
http
</server>
<features>
http
</features>
<command option="no-output">
http://%HOSTIP:%HTTPPORT/%TESTNUMBER --output-dir log -o exist%TESTNUMBER --no-clobber
</command>
<file name="log/exist%TESTNUMBER">
to stay the same
</file>
</client>
#
# Verify data after the test has been "shot"
<verify>
<file name="log/exist%TESTNUMBER">
to stay the same
</file>
<file1 name="log/exist%TESTNUMBER.1">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</file1>
</verify>
</testcase>

61
tests/data/test1683 Normal file
View File

@ -0,0 +1,61 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
--no-clobber
</keywords>
</info>
#
# Server-side
<reply>
<data nocheck="yes">
HTTP/1.0 200 OK
Connection: close
Content-Type: text/plain
Content-Length: 4
foo
</data>
</reply>
#
# Client-side
<client>
<name>
HTTP GET without clobber when 100 files already exist
</name>
<server>
http
</server>
<features>
http
</features>
<command option="no-output">
http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --no-clobber
</command>
<file name="log/exist%TESTNUMBER">
to stay the same
</file>
<precheck>
perl -e 'for my $i ((1..100)) { my $filename = "log/exist%TESTNUMBER.$i"; open(FH, ">", $filename) or die $!; print FH "to stay the same" ; close(FH) }'
# python3 -c 'for i in range(1, 101): open("log/exist%TESTNUMBER.{}".format(i), mode="w").write("to stay the same")'
</precheck>
<postcheck>
perl -e 'for my $i ((1..100)) { my $filename = "log/exist%TESTNUMBER.$i"; open(FH, "<", $filename) or die $!; (<FH> eq "to stay the same" and <FH> eq "") or die "incorrect $filename" ; close(FH) }'
# python3 -c 'for i in range(1, 101): assert open("log/exist%TESTNUMBER.{}".format(i), mode="r").read(17) == "to stay the same"'
</postcheck>
</client>
#
# Verify data after the test has been "shot"
<verify>
<errorcode>
23
</errorcode>
<file name="log/exist%TESTNUMBER">
to stay the same
</file>
</verify>
</testcase>

6
tests/manpage-scan.pl
View File

@ -6,7 +6,7 @@
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 2016 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
# Copyright (C) 2016 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
@ -146,6 +146,7 @@ my %opts = (
'--no-sessionid' => 1,
'--no-keepalive' => 1,
'--no-progress-meter' => 1,
'--no-clobber' => 1,
# pretend these options without -no exist in curl.1 and tool_listhelp.c
'--alpn' => 6,
@ -156,8 +157,9 @@ my %opts = (
'-N, --buffer' => 6,
'--sessionid' => 6,
'--progress-meter' => 6,
'--clobber' => 6,
# deprecated options do not need to be in tool_listhelp.c nor curl.1
# deprecated options do not need to be in tool_help.c nor curl.1
'--krb4' => 6,
'--ftp-ssl' => 6,
'--ftp-ssl-reqd' => 6,