diff --git a/docs/cmdline-opts/variable.md b/docs/cmdline-opts/variable.md index 9ecc3f0e71..a84078ab8e 100644 --- a/docs/cmdline-opts/variable.md +++ b/docs/cmdline-opts/variable.md @@ -36,6 +36,15 @@ the environment variable is not set, use --variable %name=content or --variable %name@content. Note that on some systems - but not all - environment variables are case insensitive. +Added in curl 8.12.0: when getting contents from a file, you can request to +get a byte range from it by appending ";[start-end]" to the filename, where +*start* and *end* are byte offsets to include from the file. For example, +asking for offset "2-10" means offset two to offset ten, including the byte +offset 10, meaning 9 bytes in total. "2-2" means a single byte at offset 2. +Not providing a second number implies to the end of the file. The start offset +cannot be larger than the end offset. Asking for a range that is outside of +the file size makes the variable contents empty. + To assign a variable using contents from another variable, use --expand-variable. Like for example assigning a new variable using contents from two other: diff --git a/src/tool_getparam.h b/src/tool_getparam.h index 7d9abbd141..90708e001c 100644 --- a/src/tool_getparam.h +++ b/src/tool_getparam.h @@ -348,6 +348,7 @@ typedef enum { PARAM_READ_ERROR, PARAM_EXPAND_ERROR, /* --expand problem */ PARAM_BLANK_STRING, + PARAM_VAR_SYNTAX, /* --variable syntax error */ PARAM_LAST } ParameterError; diff --git a/src/tool_helpers.c b/src/tool_helpers.c index 02193c3e51..aa315880ee 100644 --- a/src/tool_helpers.c +++ b/src/tool_helpers.c @@ -75,6 +75,8 @@ const char *param2text(ParameterError error) return "variable expansion failure"; case PARAM_BLANK_STRING: return "blank argument where content is expected"; + case PARAM_VAR_SYNTAX: + return "syntax error in --variable argument"; default: return "unknown error"; } diff --git a/src/tool_paramhlp.c b/src/tool_paramhlp.c index d4024e1340..5abf49d342 100644 --- a/src/tool_paramhlp.c +++ b/src/tool_paramhlp.c @@ -124,15 +124,45 @@ ParameterError file2string(char **bufp, FILE *file) return PARAM_OK; } -ParameterError file2memory(char **bufp, size_t *size, FILE *file) +static int myfseek(void *stream, curl_off_t offset, int whence) +{ +#if defined(_WIN32) && defined(USE_WIN32_LARGE_FILES) + return _fseeki64(stream, (__int64)offset, whence); +#elif defined(HAVE_FSEEKO) && defined(HAVE_DECL_FSEEKO) + return fseeko(stream, (off_t)offset, whence); +#else + if(offset > LONG_MAX) + return -1; + return fseek(stream, (long)offset, whence); +#endif +} + +ParameterError file2memory_range(char **bufp, size_t *size, FILE *file, + curl_off_t starto, curl_off_t endo) { if(file) { size_t nread; struct curlx_dynbuf dyn; + curl_off_t offset = 0; + curl_off_t throwaway = 0; + + if(starto) { + if(file != stdin) { + if(myfseek(file, starto, SEEK_SET)) + return PARAM_READ_ERROR; + offset = starto; + } + else + /* we can't seek stdin, read 'starto' bytes and throw them away */ + throwaway = starto; + } + /* The size needs to fit in an int later */ curlx_dyn_init(&dyn, MAX_FILE2MEMORY); do { char buffer[4096]; + size_t n_add; + char *ptr_add; nread = fread(buffer, 1, sizeof(buffer), file); if(ferror(file)) { curlx_dyn_free(&dyn); @@ -140,9 +170,35 @@ ParameterError file2memory(char **bufp, size_t *size, FILE *file) *bufp = NULL; return PARAM_READ_ERROR; } - if(nread) - if(curlx_dyn_addn(&dyn, buffer, nread)) - return PARAM_NO_MEM; + n_add = nread; + ptr_add = buffer; + if(nread) { + if(throwaway) { + if(throwaway >= (curl_off_t)nread) { + throwaway -= nread; + offset += nread; + n_add = 0; /* nothing to add */ + } + else { + /* append the trailing piece */ + n_add = (size_t)(nread - throwaway); + ptr_add = &buffer[throwaway]; + offset += throwaway; + throwaway = 0; + } + } + if(n_add) { + if((curl_off_t)(n_add + offset) > endo) + n_add = (size_t)(endo - offset + 1); + + if(curlx_dyn_addn(&dyn, ptr_add, n_add)) + return PARAM_NO_MEM; + + offset += n_add; + if(offset > endo) + break; + } + } } while(!feof(file)); *size = curlx_dyn_len(&dyn); *bufp = curlx_dyn_ptr(&dyn); @@ -154,6 +210,11 @@ ParameterError file2memory(char **bufp, size_t *size, FILE *file) return PARAM_OK; } +ParameterError file2memory(char **bufp, size_t *size, FILE *file) +{ + return file2memory_range(bufp, size, file, 0, CURL_OFF_T_MAX); +} + /* * Parse the string and write the long in the given address. Return PARAM_OK * on success, otherwise a parameter specific error enum. diff --git a/src/tool_paramhlp.h b/src/tool_paramhlp.h index bd703afc8c..136214bb20 100644 --- a/src/tool_paramhlp.h +++ b/src/tool_paramhlp.h @@ -37,6 +37,8 @@ ParameterError file2string(char **bufp, FILE *file); #endif ParameterError file2memory(char **bufp, size_t *size, FILE *file); +ParameterError file2memory_range(char **bufp, size_t *size, FILE *file, + curl_off_t starto, curl_off_t endo); ParameterError str2num(long *val, const char *str); ParameterError str2unum(long *val, const char *str); diff --git a/src/var.c b/src/var.c index 348c7707d6..e42b5b2c40 100644 --- a/src/var.c +++ b/src/var.c @@ -374,6 +374,8 @@ static ParameterError addvariable(struct GlobalConfig *global, return PARAM_NO_MEM; } +#define MAX_FILENAME 10000 + ParameterError setvariable(struct GlobalConfig *global, const char *input) { @@ -427,21 +429,56 @@ ParameterError setvariable(struct GlobalConfig *global, /* read from file or stdin */ FILE *file; bool use_stdin; + char *range; + struct dynbuf fname; + curl_off_t startoffset = 0; + curl_off_t endoffset = CURL_OFF_T_MAX; line++; + + Curl_dyn_init(&fname, MAX_FILENAME); + + /* is there a byte range specified? ;[num-num] */ + range = strstr(line, ";["); + if(range && ISDIGIT(range[2])) { + char *p = range; + char *endp; + if(curlx_strtoofft(&p[2], &endp, 10, &startoffset) || (*endp != '-')) + return PARAM_VAR_SYNTAX; + else { + p = endp + 1; /* pass the '-' */ + if(*p != ']') { + if(curlx_strtoofft(p, &endp, 10, &endoffset) || (*endp != ']')) + return PARAM_VAR_SYNTAX; + } + } + if(startoffset > endoffset) + return PARAM_VAR_SYNTAX; + /* create a dynbuf for the filename without the range */ + if(Curl_dyn_addn(&fname, line, (range - line))) + return PARAM_NO_MEM; + /* point to the new file name buffer */ + line = Curl_dyn_ptr(&fname); + } + use_stdin = !strcmp(line, "-"); if(use_stdin) file = stdin; else { file = fopen(line, "rb"); if(!file) { - errorf(global, "Failed to open %s", line); - return PARAM_READ_ERROR; + errorf(global, "Failed to open %s: %s", line, + strerror(errno)); + err = PARAM_READ_ERROR; } } - err = file2memory(&content, &clen, file); - /* in case of out of memory, this should fail the entire operation */ - contalloc = TRUE; - if(!use_stdin) + if(!err) { + err = file2memory_range(&content, &clen, file, startoffset, endoffset); + /* in case of out of memory, this should fail the entire operation */ + if(clen) + contalloc = TRUE; + } + Curl_dyn_free(&fname); + if(!use_stdin && file) fclose(file); if(err) return err; diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index 105108309d..6d4ddd29ae 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -109,7 +109,8 @@ test718 test719 test720 test721 test722 test723 test724 test725 test726 \ test727 test728 test729 test730 test731 test732 test733 test734 test735 \ test736 test737 test738 test739 test740 test741 test742 \ \ -test780 test781 test782 test783 \ +test780 test781 test782 test783 test784 test785 test786 test787 test788 \ +test789 \ \ test799 test800 test801 test802 test803 test804 test805 test806 test807 \ test808 test809 test810 test811 test812 test813 test814 test815 test816 \ diff --git a/tests/data/test784 b/tests/data/test784 new file mode 100644 index 0000000000..0c6acc67a8 --- /dev/null +++ b/tests/data/test784 @@ -0,0 +1,59 @@ + + + +HTTP +--variable + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +--variable with a file byte range + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[5-15]" --expand-data '{{name}}' + + +On the first Monday of the month of April, 1625, the market town of Meung + + + +# +# Verify data after the test has been "shot" + + +POST /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Content-Length: 11 +Content-Type: application/x-www-form-urlencoded + +e first Mon + + + diff --git a/tests/data/test785 b/tests/data/test785 new file mode 100644 index 0000000000..c46104a67a --- /dev/null +++ b/tests/data/test785 @@ -0,0 +1,59 @@ + + + +HTTP +--variable + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +--variable with a file byte range without end + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[5-]" --expand-data '{{name}}' + + +On the first Monday of the month of April, 1625, the market town of Meung + + + +# +# Verify data after the test has been "shot" + + +POST /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Content-Length: 69 +Content-Type: application/x-www-form-urlencoded + +e first Monday of the month of April, 1625, the market town of Meung + + + diff --git a/tests/data/test786 b/tests/data/test786 new file mode 100644 index 0000000000..5da450aae0 --- /dev/null +++ b/tests/data/test786 @@ -0,0 +1,59 @@ + + + +HTTP +--variable + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +--variable with a file byte range, reading from stdin + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@-;[5-15]" --expand-data '{{name}}' + + +On the first Monday of the month of April, 1625, the market town of Meung + + + +# +# Verify data after the test has been "shot" + + +POST /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Content-Length: 11 +Content-Type: application/x-www-form-urlencoded + +e first Mon + + + diff --git a/tests/data/test787 b/tests/data/test787 new file mode 100644 index 0000000000..23dbad8993 --- /dev/null +++ b/tests/data/test787 @@ -0,0 +1,35 @@ + + + +HTTP +--variable + + + +# +# Server-side + + + +# +# Client-side + + +http + + +--variable with a file byte range, bad range + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@&LOGDIR/fooo;[15-14]" --expand-data '{{name}}' + + + +# +# Verify data after the test has been "shot" + + +2 + + + diff --git a/tests/data/test788 b/tests/data/test788 new file mode 100644 index 0000000000..268eeb6cc9 --- /dev/null +++ b/tests/data/test788 @@ -0,0 +1,59 @@ + + + +HTTP +--variable + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +--variable with a file and single-byte byte range + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[15-15]" --expand-data '{{name}}' + + +On the first Monday of the month of April, 1625, the market town of Meung + + + +# +# Verify data after the test has been "shot" + + +POST /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Content-Length: 1 +Content-Type: application/x-www-form-urlencoded + +n + + + diff --git a/tests/data/test789 b/tests/data/test789 new file mode 100644 index 0000000000..dd36303e24 --- /dev/null +++ b/tests/data/test789 @@ -0,0 +1,58 @@ + + + +HTTP +--variable + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +--variable with a file and byte range out of file + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[75-85]" --expand-data '{{name}}' + + +On the first Monday of the month of April, 1625, the market town of Meung + + + +# +# Verify data after the test has been "shot" + + +POST /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Content-Length: 0 +Content-Type: application/x-www-form-urlencoded + + + +