luo980/curl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

curl: add --skip-existing

Browse Source 
With this option, the entire download is skipped if the selected target
filename already exists when the opertion is about to begin.

Test 994, 995 and 996 verify.

Ref: #11012
Closes #13993
This commit is contained in:
Daniel Stenberg 2024-08-04 16:14:24 +02:00
parent eec908bb6e
commit 732cb15b97
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
13 changed files with 213 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/cmdline-opts/Makefile.inc
View File

@ -253,6 +253,7 @@ DPAGES = \
show-error.md \
show-headers.md \
silent.md \
skip-existing.md \
socks4.md \
socks4a.md \
socks5-basic.md \

22
docs/cmdline-opts/skip-existing.md Normal file
View File

@ -0,0 +1,22 @@
---
c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
SPDX-License-Identifier: curl
Long: skip-existing
Help: Skip download if local file already exists
Category: curl output
Added: 8.10.0
Multi: boolean
See-also:
- output
- remote-name
- no-clobber
Example:
- --skip-existing --output local/dir/file $URL
---
# `--skip-existing`
If there is a local file present when a download is requested, the operation
is skipped. Note that curl cannot know if the local file was previously
downloaded fine, or if it is incomplete etc, it just knows if there is a
filename present in the file system or not and it skips the transfer if it is.

1
docs/options-in-versions
View File

@ -218,6 +218,7 @@
--show-error (-S) 5.9
--show-headers (-i) 4.8
--silent (-s) 4.0
--skip-existing 8.10.0
--socks4 7.15.2
--socks4a 7.18.0
--socks5 7.18.0

1
src/tool_cfgable.h
View File

@ -302,6 +302,7 @@ struct OperationConfig {
struct State state; /* for create_transfer() */
bool rm_partial; /* on error, remove partially written output
files */
bool skip_existing;
#ifdef USE_ECH
char *ech; /* Config set by --ech keywords */
char *ech_config; /* Config set by "--ech esl:" option */

4
src/tool_getparam.c
View File

@ -287,6 +287,7 @@ static const struct LongShort aliases[]= {
{"show-error", ARG_BOOL, 'S', C_SHOW_ERROR},
{"show-headers", ARG_BOOL, 'i', C_SHOW_HEADERS},
{"silent", ARG_BOOL, 's', C_SILENT},
{"skip-existing", ARG_BOOL, ' ', C_SKIP_EXISTING},
{"socks4", ARG_STRG, ' ', C_SOCKS4},
{"socks4a", ARG_STRG, ' ', C_SOCKS4A},
{"socks5", ARG_STRG, ' ', C_SOCKS5},
@ -2435,6 +2436,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
case C_SILENT: /* --silent */
global->silent = toggle;
break;
case C_SKIP_EXISTING: /* --skip-existing */
config->skip_existing = toggle;
break;
case C_SHOW_ERROR: /* --show-error */
global->showerror = toggle;
break;

1
src/tool_getparam.h
View File

@ -242,6 +242,7 @@ typedef enum {
C_SHOW_ERROR,
C_SHOW_HEADERS,
C_SILENT,
C_SKIP_EXISTING,
C_SOCKS4,
C_SOCKS4A,
C_SOCKS5,

3
src/tool_listhelp.c
View File

@ -662,6 +662,9 @@ const struct helptxt helptext[] = {
{"-s, --silent",
"Silent mode",
CURLHELP_IMPORTANT | CURLHELP_VERBOSE},
{" --skip-existing",
"Skip download if local file already exists",
CURLHELP_CURL | CURLHELP_OUTPUT},
{" --socks4 <host[:port]>",
"SOCKS4 proxy on given host + port",
CURLHELP_PROXY},

46
src/tool_operate.c
View File

@ -460,6 +460,9 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
if(per->infdopen)
close(per->infd);
if(per->skip)
goto skip;
#ifdef __VMS
if(is_vms_shell()) {
/* VMS DCL shell behavior */
@ -731,7 +734,7 @@ noretry:
curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime);
setfiletime(filetime, outs->filename, global);
}
skip:
/* Write the --write-out data before cleanup but after result is final */
if(config->writeout)
ourWriteOut(config, per, result);
@ -1197,6 +1200,15 @@ static CURLcode single_transfer(struct GlobalConfig *global,
break;
}
if(per->outfile && config->skip_existing) {
struct_stat fileinfo;
if(!stat(per->outfile, &fileinfo)) {
/* file is present */
notef(global, "skips transfer, \"%s\" exists locally",
per->outfile);
per->skip = TRUE;
}
}
if((urlnode->flags & GETOUT_USEREMOTE)
&& config->content_disposition) {
/* Our header callback MIGHT set the filename */
@ -2611,25 +2623,29 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
long delay_ms;
bool bailout = FALSE;
struct timeval start;
result = pre_transfer(global, per);
if(result)
break;
if(global->libcurl) {
result = easysrc_perform();
start = tvnow();
if(!per->skip) {
result = pre_transfer(global, per);
if(result)
break;
}
start = tvnow();
#ifdef DEBUGBUILD
if(getenv("CURL_FORBID_REUSE"))
(void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
if(global->test_event_based)
result = curl_easy_perform_ev(per->curl);
else
if(global->libcurl) {
result = easysrc_perform();
if(result)
break;
}
#ifdef DEBUGBUILD
if(getenv("CURL_FORBID_REUSE"))
(void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
if(global->test_event_based)
result = curl_easy_perform_ev(per->curl);
else
#endif
result = curl_easy_perform(per->curl);
result = curl_easy_perform(per->curl);
}
returncode = post_per_transfer(global, per, result, &retry, &delay_ms);
if(retry) {

18
src/tool_operate.h
View File

@ -44,25 +44,16 @@ struct per_transfer {
char *this_url;
unsigned int urlnum; /* the index of the given URL */
char *outfile;
bool infdopen; /* TRUE if infd needs closing */
int infd;
bool noprogress;
struct ProgressData progressbar;
struct OutStruct outs;
struct OutStruct heads;
struct OutStruct etag_save;
struct HdrCbData hdrcbdata;
long num_headers;
bool was_last_header_empty;
bool added; /* set TRUE when added to the multi handle */
time_t startat; /* when doing parallel transfers, this is a retry transfer
that has been set to sleep until this time before it
should get started (again) */
bool abort; /* when doing parallel transfers and this is TRUE then a critical
error (eg --fail-early) has occurred in another transfer and
this transfer will be aborted in the progress callback */
/* for parallel progress bar */
curl_off_t dltotal;
curl_off_t dlnow;
@ -77,6 +68,15 @@ struct per_transfer {
char *uploadfile;
char *errorbuffer; /* allocated and assigned while this is used for a
transfer */
bool infdopen; /* TRUE if infd needs closing */
bool noprogress;
bool was_last_header_empty;
bool added; /* set TRUE when added to the multi handle */
bool abort; /* when doing parallel transfers and this is TRUE then a critical
error (eg --fail-early) has occurred in another transfer and
this transfer will be aborted in the progress callback */
bool skip; /* considered already done */
};
CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]);

2
tests/data/Makefile.am
View File

@ -129,7 +129,7 @@ test952 test953 test954 test955 test956 test957 test958 test959 test960 \
test961 test962 test963 test964 test965 test966 test967 test968 test969 \
test970 test971 test972 test973 test974 test975 test976 test977 test978 \
test979 test980 test981 test982 test983 test984 test985 test986 test987 \
test988 test989 test990 test991 test992 test993 \
test988 test989 test990 test991 test992 test993 test994 test995 test996 \
\
test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 \
test1008 test1009 test1010 test1011 test1012 test1013 test1014 test1015 \

42
tests/data/test994 Normal file
View File

@ -0,0 +1,42 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
</keywords>
</info>
#
# Server-side
<reply>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
--skip-existing with globbing
</name>
<command option="no-output">
-o "%LOGDIR/#1" "http://%HOSTIP:%HTTPPORT/%TESTNUMBER/{hey,ho}" --skip-existing
</command>
<file name="%LOGDIR/hey">
content
</file>
<file2 name="%LOGDIR/ho">
content
</file2>
</client>
#
# Verify data after the test has been "shot"
<verify>
<stderr mode="text">
Note: skips transfer, "%LOGDIR/hey" exists locally
Note: skips transfer, "%LOGDIR/ho" exists locally
</stderr>
</verify>
</testcase>

56
tests/data/test995 Normal file
View File

@ -0,0 +1,56 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
</keywords>
</info>
#
# Server-side
<reply>
<data crlf="yes" nocheck="yes">
HTTP/1.1 200 OK
Date: Tue, 09 Nov 2010 14:49:00 GMT
Server: test-server/fake
Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
ETag: "21025-dc7-39462498"
Accept-Ranges: bytes
Content-Length: 6
Connection: close
Content-Type: text/html
Funny-head: yesyes
-foo-
</data>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
--skip-existing without file present
</name>
<command option="no-output,no-include">
-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<protocol crlf="yes">
GET /%TESTNUMBER HTTP/1.1
Host: %HOSTIP:%HTTPPORT
User-Agent: curl/%VERSION
Accept: */*
</protocol>
<file name="%LOGDIR/there">
-foo-
</file>
</verify>
</testcase>

41
tests/data/test996 Normal file
View File

@ -0,0 +1,41 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
</keywords>
</info>
#
# Server-side
<reply>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
--skip-existing with file present
</name>
<command option="no-output">
-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing
</command>
<file name="%LOGDIR/there">
content
</file>
</client>
#
# Verify data after the test has been "shot"
<verify>
<stderr mode="text">
Note: skips transfer, "%LOGDIR/there" exists locally
</stderr>
<file name="%LOGDIR/there">
content
</file>
</verify>
</testcase>