luo980/curl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib: client writer, part 2, accounting + logging

Browse Source 
This PR has these changes:

Renaming of unencode_* to cwriter, e.g. client writers
- documentation of sendf.h functions
- move max decode stack checks back to content_encoding.c
- define writer phase which was used as order before
- introduce phases for monitoring inbetween decode phases
- offering default implementations for init/write/close

Add type paramter to client writer's do_write()
- always pass all writes through the writer stack
- writers who only care about BODY data will pass other writes unchanged

add RAW and PROTOCOL client writers
- RAW used for Curl_debug() logging of CURLINFO_DATA_IN
- PROTOCOL used for updates to data->req.bytecount, max_filesize checks and
  Curl_pgrsSetDownloadCounter()
- remove all updates of data->req.bytecount and calls to
  Curl_pgrsSetDownloadCounter() and Curl_debug() from other code
- adjust test457 expected output to no longer see the excess write

Closes #12184
This commit is contained in:
Stefan Eissing 2023-10-23 10:33:07 +02:00 committed by Daniel Stenberg
parent 2b16b86bb6
commit ad051e1cbe
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
20 changed files with 459 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
lib/c-hyper.c
View File

@ -172,17 +172,15 @@ static int hyper_each_header(void *userdata,
Curl_debug(data, CURLINFO_HEADER_IN, headp, len); Curl_debug(data, CURLINFO_HEADER_IN, headp, len);
if(!data->state.hconnect || !data->set.suppress_connect_headers) { writetype = CLIENTWRITE_HEADER;
writetype = CLIENTWRITE_HEADER; if(data->state.hconnect)
if(data->state.hconnect) writetype |= CLIENTWRITE_CONNECT;
writetype |= CLIENTWRITE_CONNECT; if(data->req.httpcode/100 == 1)
if(data->req.httpcode/100 == 1) writetype |= CLIENTWRITE_1XX;
writetype |= CLIENTWRITE_1XX; result = Curl_client_write(data, writetype, headp, len);
result = Curl_client_write(data, writetype, headp, len); if(result) {
if(result) { data->state.hresult = CURLE_ABORTED_BY_CALLBACK;
data->state.hresult = CURLE_ABORTED_BY_CALLBACK; return HYPER_ITER_BREAK;
return HYPER_ITER_BREAK;
}
} }
result = Curl_bump_headersize(data, len, FALSE); result = Curl_bump_headersize(data, len, FALSE);
@ -201,7 +199,7 @@ static int hyper_body_chunk(void *userdata, const hyper_buf *chunk)
struct SingleRequest *k = &data->req; struct SingleRequest *k = &data->req;
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
if(0 == k->bodywrites++) { if(0 == k->bodywrites) {
bool done = FALSE; bool done = FALSE;
#if defined(USE_NTLM) #if defined(USE_NTLM)
struct connectdata *conn = data->conn; struct connectdata *conn = data->conn;
@ -241,11 +239,6 @@ static int hyper_body_chunk(void *userdata, const hyper_buf *chunk)
return HYPER_ITER_BREAK; return HYPER_ITER_BREAK;
} }
} }
if(k->ignorebody)
return HYPER_ITER_CONTINUE;
if(0 == len)
return HYPER_ITER_CONTINUE;
Curl_debug(data, CURLINFO_DATA_IN, buf, len);
result = Curl_client_write(data, CLIENTWRITE_BODY, buf, len); result = Curl_client_write(data, CLIENTWRITE_BODY, buf, len);
if(result) { if(result) {
@ -253,12 +246,6 @@ static int hyper_body_chunk(void *userdata, const hyper_buf *chunk)
return HYPER_ITER_BREAK; return HYPER_ITER_BREAK;
} }
data->req.bytecount += len;
result = Curl_pgrsSetDownloadCounter(data, data->req.bytecount);
if(result) {
data->state.hresult = result;
return HYPER_ITER_BREAK;
}
return HYPER_ITER_CONTINUE; return HYPER_ITER_CONTINUE;
} }
@ -310,13 +297,14 @@ static CURLcode status_line(struct Curl_easy *data,
Curl_debug(data, CURLINFO_HEADER_IN, Curl_dyn_ptr(&data->state.headerb), Curl_debug(data, CURLINFO_HEADER_IN, Curl_dyn_ptr(&data->state.headerb),
len); len);
if(!data->state.hconnect || !data->set.suppress_connect_headers) { writetype = CLIENTWRITE_HEADER|CLIENTWRITE_STATUS;
writetype = CLIENTWRITE_HEADER|CLIENTWRITE_STATUS; if(data->state.hconnect)
result = Curl_client_write(data, writetype, writetype |= CLIENTWRITE_CONNECT;
Curl_dyn_ptr(&data->state.headerb), len); result = Curl_client_write(data, writetype,
if(result) Curl_dyn_ptr(&data->state.headerb), len);
return result; if(result)
} return result;
result = Curl_bump_headersize(data, len, FALSE); result = Curl_bump_headersize(data, len, FALSE);
return result; return result;
} }

17
lib/cf-h1-proxy.c
View File

@ -374,7 +374,7 @@ static CURLcode recv_CONNECT_resp(struct Curl_cfilter *cf,
curl_socket_t tunnelsocket = Curl_conn_cf_get_socket(cf, data); curl_socket_t tunnelsocket = Curl_conn_cf_get_socket(cf, data);
char *linep; char *linep;
size_t perline; size_t perline;
int error; int error, writetype;
#define SELECT_OK 0 #define SELECT_OK 0
#define SELECT_ERROR 1 #define SELECT_ERROR 1
@ -467,15 +467,12 @@ static CURLcode recv_CONNECT_resp(struct Curl_cfilter *cf,
/* output debug if that is requested */ /* output debug if that is requested */
Curl_debug(data, CURLINFO_HEADER_IN, linep, perline); Curl_debug(data, CURLINFO_HEADER_IN, linep, perline);
if(!data->set.suppress_connect_headers) { /* send the header to the callback */
/* send the header to the callback */ writetype = CLIENTWRITE_HEADER | CLIENTWRITE_CONNECT |
int writetype = CLIENTWRITE_HEADER | CLIENTWRITE_CONNECT | (ts->headerlines == 1 ? CLIENTWRITE_STATUS : 0);
(ts->headerlines == 1 ? CLIENTWRITE_STATUS : 0); result = Curl_client_write(data, writetype, linep, perline);
if(result)
result = Curl_client_write(data, writetype, linep, perline); return result;
if(result)
return result;
}
result = Curl_bump_headersize(data, perline, TRUE); result = Curl_bump_headersize(data, perline, TRUE);
if(result) if(result)

242
lib/content_encoding.c
View File

@ -63,6 +63,9 @@
#ifndef CURL_DISABLE_HTTP #ifndef CURL_DISABLE_HTTP
/* allow no more than 5 "chained" compression steps */
#define MAX_ENCODE_STACK 5
#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */ #define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
@ -95,7 +98,7 @@ typedef enum {
/* Deflate and gzip writer. */ /* Deflate and gzip writer. */
struct zlib_writer { struct zlib_writer {
struct contenc_writer super; struct Curl_cwriter super;
zlibInitState zlib_init; /* zlib init state */ zlibInitState zlib_init; /* zlib init state */
uInt trailerlen; /* Remaining trailer byte count. */ uInt trailerlen; /* Remaining trailer byte count. */
z_stream z; /* State structure for zlib. */ z_stream z; /* State structure for zlib. */
@ -171,7 +174,7 @@ static CURLcode process_trailer(struct Curl_easy *data,
} }
static CURLcode inflate_stream(struct Curl_easy *data, static CURLcode inflate_stream(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
zlibInitState started) zlibInitState started)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
@ -196,7 +199,7 @@ static CURLcode inflate_stream(struct Curl_easy *data,
return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY); return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);
/* because the buffer size is fixed, iteratively decompress and transfer to /* because the buffer size is fixed, iteratively decompress and transfer to
the client via downstream_write function. */ the client via next_write function. */
while(!done) { while(!done) {
int status; /* zlib status */ int status; /* zlib status */
done = TRUE; done = TRUE;
@ -217,7 +220,7 @@ static CURLcode inflate_stream(struct Curl_easy *data,
if(z->avail_out != DSIZ) { if(z->avail_out != DSIZ) {
if(status == Z_OK || status == Z_STREAM_END) { if(status == Z_OK || status == Z_STREAM_END) {
zp->zlib_init = started; /* Data started. */ zp->zlib_init = started; /* Data started. */
result = Curl_unencode_write(data, writer->downstream, decomp, result = Curl_cwriter_write(data, writer->next, type, decomp,
DSIZ - z->avail_out); DSIZ - z->avail_out);
if(result) { if(result) {
exit_zlib(data, z, &zp->zlib_init, result); exit_zlib(data, z, &zp->zlib_init, result);
@ -274,8 +277,8 @@ static CURLcode inflate_stream(struct Curl_easy *data,
/* Deflate handler. */ /* Deflate handler. */
static CURLcode deflate_init_writer(struct Curl_easy *data, static CURLcode deflate_do_init(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */ z_stream *z = &zp->z; /* zlib state structure */
@ -290,13 +293,16 @@ static CURLcode deflate_init_writer(struct Curl_easy *data,
return CURLE_OK; return CURLE_OK;
} }
static CURLcode deflate_unencode_write(struct Curl_easy *data, static CURLcode deflate_do_write(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes) const char *buf, size_t nbytes)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */ z_stream *z = &zp->z; /* zlib state structure */
if(!(type & CLIENTWRITE_BODY))
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
/* Set the compressed input when this function is called */ /* Set the compressed input when this function is called */
z->next_in = (Bytef *) buf; z->next_in = (Bytef *) buf;
z->avail_in = (uInt) nbytes; z->avail_in = (uInt) nbytes;
@ -305,11 +311,11 @@ static CURLcode deflate_unencode_write(struct Curl_easy *data,
return process_trailer(data, zp); return process_trailer(data, zp);
/* Now uncompress the data */ /* Now uncompress the data */
return inflate_stream(data, writer, ZLIB_INFLATING); return inflate_stream(data, writer, type, ZLIB_INFLATING);
} }
static void deflate_close_writer(struct Curl_easy *data, static void deflate_do_close(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */ z_stream *z = &zp->z; /* zlib state structure */
@ -317,19 +323,19 @@ static void deflate_close_writer(struct Curl_easy *data,
exit_zlib(data, z, &zp->zlib_init, CURLE_OK); exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
} }
static const struct content_encoding deflate_encoding = { static const struct Curl_cwtype deflate_encoding = {
"deflate", "deflate",
NULL, NULL,
deflate_init_writer, deflate_do_init,
deflate_unencode_write, deflate_do_write,
deflate_close_writer, deflate_do_close,
sizeof(struct zlib_writer) sizeof(struct zlib_writer)
}; };
/* Gzip handler. */ /* Gzip handler. */
static CURLcode gzip_init_writer(struct Curl_easy *data, static CURLcode gzip_do_init(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */ z_stream *z = &zp->z; /* zlib state structure */
@ -441,19 +447,22 @@ static enum {
} }
#endif #endif
static CURLcode gzip_unencode_write(struct Curl_easy *data, static CURLcode gzip_do_write(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes) const char *buf, size_t nbytes)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */ z_stream *z = &zp->z; /* zlib state structure */
if(!(type & CLIENTWRITE_BODY))
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(zp->zlib_init == ZLIB_INIT_GZIP) { if(zp->zlib_init == ZLIB_INIT_GZIP) {
/* Let zlib handle the gzip decompression entirely */ /* Let zlib handle the gzip decompression entirely */
z->next_in = (Bytef *) buf; z->next_in = (Bytef *) buf;
z->avail_in = (uInt) nbytes; z->avail_in = (uInt) nbytes;
/* Now uncompress the data */ /* Now uncompress the data */
return inflate_stream(data, writer, ZLIB_INIT_GZIP); return inflate_stream(data, writer, type, ZLIB_INIT_GZIP);
} }
#ifndef OLD_ZLIB_SUPPORT #ifndef OLD_ZLIB_SUPPORT
@ -565,12 +574,12 @@ static CURLcode gzip_unencode_write(struct Curl_easy *data,
} }
/* We've parsed the header, now uncompress the data */ /* We've parsed the header, now uncompress the data */
return inflate_stream(data, writer, ZLIB_GZIP_INFLATING); return inflate_stream(data, writer, type, ZLIB_GZIP_INFLATING);
#endif #endif
} }
static void gzip_close_writer(struct Curl_easy *data, static void gzip_do_close(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct zlib_writer *zp = (struct zlib_writer *) writer; struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */ z_stream *z = &zp->z; /* zlib state structure */
@ -578,12 +587,12 @@ static void gzip_close_writer(struct Curl_easy *data,
exit_zlib(data, z, &zp->zlib_init, CURLE_OK); exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
} }
static const struct content_encoding gzip_encoding = { static const struct Curl_cwtype gzip_encoding = {
"gzip", "gzip",
"x-gzip", "x-gzip",
gzip_init_writer, gzip_do_init,
gzip_unencode_write, gzip_do_write,
gzip_close_writer, gzip_do_close,
sizeof(struct zlib_writer) sizeof(struct zlib_writer)
}; };
@ -593,7 +602,7 @@ static const struct content_encoding gzip_encoding = {
#ifdef HAVE_BROTLI #ifdef HAVE_BROTLI
/* Brotli writer. */ /* Brotli writer. */
struct brotli_writer { struct brotli_writer {
struct contenc_writer super; struct Curl_cwriter super;
BrotliDecoderState *br; /* State structure for brotli. */ BrotliDecoderState *br; /* State structure for brotli. */
}; };
@ -635,8 +644,8 @@ static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
return CURLE_WRITE_ERROR; return CURLE_WRITE_ERROR;
} }
static CURLcode brotli_init_writer(struct Curl_easy *data, static CURLcode brotli_do_init(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct brotli_writer *bp = (struct brotli_writer *) writer; struct brotli_writer *bp = (struct brotli_writer *) writer;
(void) data; (void) data;
@ -645,8 +654,8 @@ static CURLcode brotli_init_writer(struct Curl_easy *data,
return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY; return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
} }
static CURLcode brotli_unencode_write(struct Curl_easy *data, static CURLcode brotli_do_write(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes) const char *buf, size_t nbytes)
{ {
struct brotli_writer *bp = (struct brotli_writer *) writer; struct brotli_writer *bp = (struct brotli_writer *) writer;
@ -657,6 +666,9 @@ static CURLcode brotli_unencode_write(struct Curl_easy *data,
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
if(!(type & CLIENTWRITE_BODY))
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!bp->br) if(!bp->br)
return CURLE_WRITE_ERROR; /* Stream already ended. */ return CURLE_WRITE_ERROR; /* Stream already ended. */
@ -670,7 +682,7 @@ static CURLcode brotli_unencode_write(struct Curl_easy *data,
dstleft = DSIZ; dstleft = DSIZ;
r = BrotliDecoderDecompressStream(bp->br, r = BrotliDecoderDecompressStream(bp->br,
&nbytes, &src, &dstleft, &dst, NULL); &nbytes, &src, &dstleft, &dst, NULL);
result = Curl_unencode_write(data, writer->downstream, result = Curl_cwriter_write(data, writer->next, type,
decomp, DSIZ - dstleft); decomp, DSIZ - dstleft);
if(result) if(result)
break; break;
@ -693,8 +705,8 @@ static CURLcode brotli_unencode_write(struct Curl_easy *data,
return result; return result;
} }
static void brotli_close_writer(struct Curl_easy *data, static void brotli_do_close(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct brotli_writer *bp = (struct brotli_writer *) writer; struct brotli_writer *bp = (struct brotli_writer *) writer;
@ -706,12 +718,12 @@ static void brotli_close_writer(struct Curl_easy *data,
} }
} }
static const struct content_encoding brotli_encoding = { static const struct Curl_cwtype brotli_encoding = {
"br", "br",
NULL, NULL,
brotli_init_writer, brotli_do_init,
brotli_unencode_write, brotli_do_write,
brotli_close_writer, brotli_do_close,
sizeof(struct brotli_writer) sizeof(struct brotli_writer)
}; };
#endif #endif
@ -720,13 +732,13 @@ static const struct content_encoding brotli_encoding = {
#ifdef HAVE_ZSTD #ifdef HAVE_ZSTD
/* Zstd writer. */ /* Zstd writer. */
struct zstd_writer { struct zstd_writer {
struct contenc_writer super; struct Curl_cwriter super;
ZSTD_DStream *zds; /* State structure for zstd. */ ZSTD_DStream *zds; /* State structure for zstd. */
void *decomp; void *decomp;
}; };
static CURLcode zstd_init_writer(struct Curl_easy *data, static CURLcode zstd_do_init(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct zstd_writer *zp = (struct zstd_writer *) writer; struct zstd_writer *zp = (struct zstd_writer *) writer;
@ -737,8 +749,8 @@ static CURLcode zstd_init_writer(struct Curl_easy *data,
return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY; return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
} }
static CURLcode zstd_unencode_write(struct Curl_easy *data, static CURLcode zstd_do_write(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes) const char *buf, size_t nbytes)
{ {
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
@ -747,6 +759,9 @@ static CURLcode zstd_unencode_write(struct Curl_easy *data,
ZSTD_outBuffer out; ZSTD_outBuffer out;
size_t errorCode; size_t errorCode;
if(!(type & CLIENTWRITE_BODY))
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!zp->decomp) { if(!zp->decomp) {
zp->decomp = malloc(DSIZ); zp->decomp = malloc(DSIZ);
if(!zp->decomp) if(!zp->decomp)
@ -766,7 +781,7 @@ static CURLcode zstd_unencode_write(struct Curl_easy *data,
return CURLE_BAD_CONTENT_ENCODING; return CURLE_BAD_CONTENT_ENCODING;
} }
if(out.pos > 0) { if(out.pos > 0) {
result = Curl_unencode_write(data, writer->downstream, result = Curl_cwriter_write(data, writer->next, type,
zp->decomp, out.pos); zp->decomp, out.pos);
if(result) if(result)
break; break;
@ -778,8 +793,8 @@ static CURLcode zstd_unencode_write(struct Curl_easy *data,
return result; return result;
} }
static void zstd_close_writer(struct Curl_easy *data, static void zstd_do_close(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
struct zstd_writer *zp = (struct zstd_writer *) writer; struct zstd_writer *zp = (struct zstd_writer *) writer;
@ -795,52 +810,30 @@ static void zstd_close_writer(struct Curl_easy *data,
} }
} }
static const struct content_encoding zstd_encoding = { static const struct Curl_cwtype zstd_encoding = {
"zstd", "zstd",
NULL, NULL,
zstd_init_writer, zstd_do_init,
zstd_unencode_write, zstd_do_write,
zstd_close_writer, zstd_do_close,
sizeof(struct zstd_writer) sizeof(struct zstd_writer)
}; };
#endif #endif
/* Identity handler. */ /* Identity handler. */
static CURLcode identity_init_writer(struct Curl_easy *data, static const struct Curl_cwtype identity_encoding = {
struct contenc_writer *writer)
{
(void)data;
(void)writer;
return CURLE_OK;
}
static CURLcode identity_unencode_write(struct Curl_easy *data,
struct contenc_writer *writer,
const char *buf, size_t nbytes)
{
return Curl_unencode_write(data, writer->downstream, buf, nbytes);
}
static void identity_close_writer(struct Curl_easy *data,
struct contenc_writer *writer)
{
(void) data;
(void) writer;
}
static const struct content_encoding identity_encoding = {
"identity", "identity",
"none", "none",
identity_init_writer, Curl_cwriter_def_init,
identity_unencode_write, Curl_cwriter_def_write,
identity_close_writer, Curl_cwriter_def_close,
sizeof(struct contenc_writer) sizeof(struct Curl_cwriter)
}; };
/* supported content encodings table. */ /* supported content encodings table. */
static const struct content_encoding * const encodings[] = { static const struct Curl_cwtype * const encodings[] = {
&identity_encoding, &identity_encoding,
#ifdef HAVE_LIBZ #ifdef HAVE_LIBZ
&deflate_encoding, &deflate_encoding,
@ -860,8 +853,8 @@ static const struct content_encoding * const encodings[] = {
char *Curl_all_content_encodings(void) char *Curl_all_content_encodings(void)
{ {
size_t len = 0; size_t len = 0;
const struct content_encoding * const *cep; const struct Curl_cwtype * const *cep;
const struct content_encoding *ce; const struct Curl_cwtype *ce;
char *ace; char *ace;
for(cep = encodings; *cep; cep++) { for(cep = encodings; *cep; cep++) {
@ -893,16 +886,16 @@ char *Curl_all_content_encodings(void)
/* Deferred error dummy writer. */ /* Deferred error dummy writer. */
static CURLcode error_init_writer(struct Curl_easy *data, static CURLcode error_do_init(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
(void)data; (void)data;
(void)writer; (void)writer;
return CURLE_OK; return CURLE_OK;
} }
static CURLcode error_unencode_write(struct Curl_easy *data, static CURLcode error_do_write(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes) const char *buf, size_t nbytes)
{ {
char *all = Curl_all_content_encodings(); char *all = Curl_all_content_encodings();
@ -911,6 +904,9 @@ static CURLcode error_unencode_write(struct Curl_easy *data,
(void) buf; (void) buf;
(void) nbytes; (void) nbytes;
if(!(type & CLIENTWRITE_BODY))
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!all) if(!all)
return CURLE_OUT_OF_MEMORY; return CURLE_OUT_OF_MEMORY;
failf(data, "Unrecognized content encoding type. " failf(data, "Unrecognized content encoding type. "
@ -919,43 +915,30 @@ static CURLcode error_unencode_write(struct Curl_easy *data,
return CURLE_BAD_CONTENT_ENCODING; return CURLE_BAD_CONTENT_ENCODING;
} }
static void error_close_writer(struct Curl_easy *data, static void error_do_close(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
(void) data; (void) data;
(void) writer; (void) writer;
} }
static const struct content_encoding error_encoding = { static const struct Curl_cwtype error_writer = {
"ce-error",
NULL, NULL,
NULL, error_do_init,
error_init_writer, error_do_write,
error_unencode_write, error_do_close,
error_close_writer, sizeof(struct Curl_cwriter)
sizeof(struct contenc_writer)
}; };
/* Write data using an unencoding writer stack. "nbytes" is not
allowed to be 0. */
CURLcode Curl_unencode_write(struct Curl_easy *data,
struct contenc_writer *writer,
const char *buf, size_t nbytes)
{
if(!nbytes)
return CURLE_OK;
if(!writer)
return CURLE_WRITE_ERROR;
return writer->handler->unencode_write(data, writer, buf, nbytes);
}
/* Find the content encoding by name. */ /* Find the content encoding by name. */
static const struct content_encoding *find_encoding(const char *name, static const struct Curl_cwtype *find_encoding(const char *name,
size_t len) size_t len)
{ {
const struct content_encoding * const *cep; const struct Curl_cwtype * const *cep;
for(cep = encodings; *cep; cep++) { for(cep = encodings; *cep; cep++) {
const struct content_encoding *ce = *cep; const struct Curl_cwtype *ce = *cep;
if((strncasecompare(name, ce->name, len) && !ce->name[len]) || if((strncasecompare(name, ce->name, len) && !ce->name[len]) ||
(ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len])) (ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len]))
return ce; return ce;
@ -969,7 +952,8 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
const char *enclist, int is_transfer) const char *enclist, int is_transfer)
{ {
struct SingleRequest *k = &data->req; struct SingleRequest *k = &data->req;
unsigned int order = is_transfer? 2: 1; Curl_cwriter_phase phase = is_transfer?
CURL_CW_TRANSFER_DECODE:CURL_CW_CONTENT_DECODE;
CURLcode result; CURLcode result;
do { do {
@ -992,23 +976,32 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
Curl_httpchunk_init(data); /* init our chunky engine. */ Curl_httpchunk_init(data); /* init our chunky engine. */
} }
else if(namelen) { else if(namelen) {
const struct content_encoding *encoding; const struct Curl_cwtype *cwt;
struct contenc_writer *writer; struct Curl_cwriter *writer;
if(is_transfer && !data->set.http_transfer_encoding)
if((is_transfer && !data->set.http_transfer_encoding) ||
(!is_transfer && data->set.http_ce_skip)) {
/* not requested, ignore */ /* not requested, ignore */
return CURLE_OK; return CURLE_OK;
}
encoding = find_encoding(name, namelen); if(Curl_cwriter_count(data, phase) + 1 >= MAX_ENCODE_STACK) {
if(!encoding) failf(data, "Reject response due to more than %u content encodings",
encoding = &error_encoding; /* Defer error at stack use. */ MAX_ENCODE_STACK);
return CURLE_BAD_CONTENT_ENCODING;
}
result = Curl_client_create_writer(&writer, data, encoding, order); cwt = find_encoding(name, namelen);
if(!cwt)
cwt = &error_writer; /* Defer error at use. */
result = Curl_cwriter_create(&writer, data, cwt, phase);
if(result) if(result)
return result; return result;
result = Curl_client_add_writer(data, writer); result = Curl_cwriter_add(data, writer);
if(result) { if(result) {
Curl_client_free_writer(data, writer); Curl_cwriter_free(data, writer);
return result; return result;
} }
} }
@ -1028,17 +1021,6 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
return CURLE_NOT_BUILT_IN; return CURLE_NOT_BUILT_IN;
} }
CURLcode Curl_unencode_write(struct Curl_easy *data,
struct contenc_writer *writer,
const char *buf, size_t nbytes)
{
(void) data;
(void) writer;
(void) buf;
(void) nbytes;
return CURLE_NOT_BUILT_IN;
}
char *Curl_all_content_encodings(void) char *Curl_all_content_encodings(void)
{ {
return strdup(CONTENT_ENCODING_DEFAULT); /* Satisfy caller. */ return strdup(CONTENT_ENCODING_DEFAULT); /* Satisfy caller. */

7
lib/content_encoding.h
View File

@ -25,15 +25,10 @@
***************************************************************************/ ***************************************************************************/
#include "curl_setup.h" #include "curl_setup.h"
struct contenc_writer; struct Curl_cwriter;
char *Curl_all_content_encodings(void); char *Curl_all_content_encodings(void);
CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
const char *enclist, int is_transfer); const char *enclist, int is_transfer);
CURLcode Curl_unencode_write(struct Curl_easy *data,
struct contenc_writer *writer,
const char *buf, size_t nbytes);
void Curl_unencode_cleanup(struct Curl_easy *data);
#endif /* HEADER_CURL_CONTENT_ENCODING_H */ #endif /* HEADER_CURL_CONTENT_ENCODING_H */

6
lib/file.c
View File

@ -414,7 +414,6 @@ static CURLcode file_do(struct Curl_easy *data, bool *done)
bool size_known; bool size_known;
bool fstated = FALSE; bool fstated = FALSE;
char *buf = data->state.buffer; char *buf = data->state.buffer;
curl_off_t bytecount = 0;
int fd; int fd;
struct FILEPROTO *file; struct FILEPROTO *file;
@ -563,7 +562,6 @@ static CURLcode file_do(struct Curl_easy *data, bool *done)
if(nread <= 0 || (size_known && (expected_size == 0))) if(nread <= 0 || (size_known && (expected_size == 0)))
break; break;
bytecount += nread;
if(size_known) if(size_known)
expected_size -= nread; expected_size -= nread;
@ -571,10 +569,6 @@ static CURLcode file_do(struct Curl_easy *data, bool *done)
if(result) if(result)
return result; return result;
result = Curl_pgrsSetDownloadCounter(data, bytecount);
if(result)
return result;
if(Curl_pgrsUpdate(data)) if(Curl_pgrsUpdate(data))
result = CURLE_ABORTED_BY_CALLBACK; result = CURLE_ABORTED_BY_CALLBACK;
else else

2
lib/imap.c
View File

@ -1194,8 +1194,6 @@ static CURLcode imap_state_fetch_resp(struct Curl_easy *data,
if(result) if(result)
return result; return result;
data->req.bytecount += chunk;
infof(data, "Written %zu bytes, %" CURL_FORMAT_CURL_OFF_TU infof(data, "Written %zu bytes, %" CURL_FORMAT_CURL_OFF_TU
" bytes are left for transfer", chunk, size - chunk); " bytes are left for transfer", chunk, size - chunk);

16
lib/ldap.c
View File

@ -313,7 +313,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
int ldap_ssl = 0; int ldap_ssl = 0;
char *val_b64 = NULL; char *val_b64 = NULL;
size_t val_b64_sz = 0; size_t val_b64_sz = 0;
curl_off_t dlsize = 0;
#ifdef LDAP_OPT_NETWORK_TIMEOUT #ifdef LDAP_OPT_NETWORK_TIMEOUT
struct timeval ldap_timeout = {10, 0}; /* 10 sec connection/search timeout */ struct timeval ldap_timeout = {10, 0}; /* 10 sec connection/search timeout */
#endif #endif
@ -535,6 +534,7 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
goto quit; goto quit;
} }
Curl_pgrsSetDownloadCounter(data, 0);
rc = ldap_search_s(server, ludp->lud_dn, ludp->lud_scope, rc = ldap_search_s(server, ludp->lud_dn, ludp->lud_scope,
ludp->lud_filter, ludp->lud_attrs, 0, &ldapmsg); ludp->lud_filter, ludp->lud_attrs, 0, &ldapmsg);
@ -596,8 +596,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
goto quit; goto quit;
} }
dlsize += name_len + 5;
FREE_ON_WINLDAP(name); FREE_ON_WINLDAP(name);
ldap_memfree(dn); ldap_memfree(dn);
} }
@ -659,8 +657,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
goto quit; goto quit;
} }
dlsize += attr_len + 3;
if((attr_len > 7) && if((attr_len > 7) &&
(strcmp(";binary", attr + (attr_len - 7)) == 0)) { (strcmp(";binary", attr + (attr_len - 7)) == 0)) {
/* Binary attribute, encode to base64. */ /* Binary attribute, encode to base64. */
@ -689,8 +685,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
goto quit; goto quit;
} }
dlsize += val_b64_sz;
} }
} }
else { else {
@ -705,8 +699,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
goto quit; goto quit;
} }
dlsize += vals[i]->bv_len;
} }
result = Curl_client_write(data, CLIENTWRITE_BODY, (char *)"\n", 1); result = Curl_client_write(data, CLIENTWRITE_BODY, (char *)"\n", 1);
@ -719,8 +711,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
goto quit; goto quit;
} }
dlsize++;
} }
/* Free memory used to store values */ /* Free memory used to store values */
@ -734,10 +724,6 @@ static CURLcode ldap_do(struct Curl_easy *data, bool *done)
result = Curl_client_write(data, CLIENTWRITE_BODY, (char *)"\n", 1); result = Curl_client_write(data, CLIENTWRITE_BODY, (char *)"\n", 1);
if(result) if(result)
goto quit; goto quit;
dlsize++;
result = Curl_pgrsSetDownloadCounter(data, dlsize);
if(result)
goto quit;
} }
if(ber) if(ber)

6
lib/mqtt.c
View File

@ -677,7 +677,6 @@ MQTT_SUBACK_COMING:
/* FALLTHROUGH */ /* FALLTHROUGH */
case MQTT_PUB_REMAIN: { case MQTT_PUB_REMAIN: {
/* read rest of packet, but no more. Cap to buffer size */ /* read rest of packet, but no more. Cap to buffer size */
struct SingleRequest *k = &data->req;
size_t rest = mq->npacket; size_t rest = mq->npacket;
if(rest > (size_t)data->set.buffer_size) if(rest > (size_t)data->set.buffer_size)
rest = (size_t)data->set.buffer_size; rest = (size_t)data->set.buffer_size;
@ -693,13 +692,8 @@ MQTT_SUBACK_COMING:
result = CURLE_PARTIAL_FILE; result = CURLE_PARTIAL_FILE;
goto end; goto end;
} }
Curl_debug(data, CURLINFO_DATA_IN, (char *)pkt, (size_t)nread);
mq->npacket -= nread; mq->npacket -= nread;
k->bytecount += nread;
result = Curl_pgrsSetDownloadCounter(data, k->bytecount);
if(result)
goto end;
/* if QoS is set, message contains packet id */ /* if QoS is set, message contains packet id */

6
lib/openldap.c
View File

@ -953,18 +953,12 @@ static CURLcode client_write(struct Curl_easy *data,
if(!len && plen && prefix[plen - 1] == ' ') if(!len && plen && prefix[plen - 1] == ' ')
plen--; plen--;
result = Curl_client_write(data, CLIENTWRITE_BODY, (char *) prefix, plen); result = Curl_client_write(data, CLIENTWRITE_BODY, (char *) prefix, plen);
if(!result)
data->req.bytecount += plen;
} }
if(!result && value) { if(!result && value) {
result = Curl_client_write(data, CLIENTWRITE_BODY, (char *) value, len); result = Curl_client_write(data, CLIENTWRITE_BODY, (char *) value, len);
if(!result)
data->req.bytecount += len;
} }
if(!result && suffix) { if(!result && suffix) {
result = Curl_client_write(data, CLIENTWRITE_BODY, (char *) suffix, slen); result = Curl_client_write(data, CLIENTWRITE_BODY, (char *) suffix, slen);
if(!result)
data->req.bytecount += slen;
} }
return result; return result;
} }

6
lib/progress.c
View File

@ -319,12 +319,6 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize,
*/ */
CURLcode Curl_pgrsSetDownloadCounter(struct Curl_easy *data, curl_off_t size) CURLcode Curl_pgrsSetDownloadCounter(struct Curl_easy *data, curl_off_t size)
{ {
if(data->set.max_filesize && (size > data->set.max_filesize)) {
failf(data, "Exceeded the maximum allowed file size "
"(%" CURL_FORMAT_CURL_OFF_T ")",
data->set.max_filesize);
return CURLE_FILESIZE_EXCEEDED;
}
data->progress.downloaded = size; data->progress.downloaded = size;
return CURLE_OK; return CURLE_OK;
} }

1
lib/rtsp.c
View File

@ -655,7 +655,6 @@ static CURLcode rtsp_filter_rtp(struct Curl_easy *data,
* to write it directly as BODY data */ * to write it directly as BODY data */
result = Curl_client_write(data, CLIENTWRITE_BODY, result = Curl_client_write(data, CLIENTWRITE_BODY,
Curl_dyn_ptr(&rtspc->buf), 1); Curl_dyn_ptr(&rtspc->buf), 1);
++data->req.bytecount;
Curl_dyn_free(&rtspc->buf); Curl_dyn_free(&rtspc->buf);
if(result) if(result)
goto out; goto out;

265
lib/sendf.c
View File

@ -50,6 +50,7 @@
#include "strdup.h" #include "strdup.h"
#include "http2.h" #include "http2.h"
#include "headers.h" #include "headers.h"
#include "progress.h"
#include "ws.h" #include "ws.h"
/* The last 3 #include files should be in this order */ /* The last 3 #include files should be in this order */
@ -57,6 +58,9 @@
#include "curl_memory.h" #include "curl_memory.h"
#include "memdebug.h" #include "memdebug.h"
static CURLcode do_init_stack(struct Curl_easy *data);
#if defined(CURL_DO_LINEEND_CONV) && !defined(CURL_DISABLE_FTP) #if defined(CURL_DO_LINEEND_CONV) && !defined(CURL_DISABLE_FTP)
/* /*
* convert_lineends() changes CRLF (\r\n) end-of-line markers to a single LF * convert_lineends() changes CRLF (\r\n) end-of-line markers to a single LF
@ -385,17 +389,17 @@ static CURLcode chop_write(struct Curl_easy *data,
the future to leave the original data alone. the future to leave the original data alone.
*/ */
CURLcode Curl_client_write(struct Curl_easy *data, CURLcode Curl_client_write(struct Curl_easy *data,
int type, int type, char *buf, size_t blen)
char *ptr,
size_t len)
{ {
CURLcode result;
#if !defined(CURL_DISABLE_FTP) && defined(CURL_DO_LINEEND_CONV) #if !defined(CURL_DISABLE_FTP) && defined(CURL_DO_LINEEND_CONV)
/* FTP data may need conversion. */ /* FTP data may need conversion. */
if((type & CLIENTWRITE_BODY) && if((type & CLIENTWRITE_BODY) &&
(data->conn->handler->protocol & PROTO_FAMILY_FTP) && (data->conn->handler->protocol & PROTO_FAMILY_FTP) &&
data->conn->proto.ftpc.transfertype == 'A') { data->conn->proto.ftpc.transfertype == 'A') {
/* convert end-of-line markers */ /* convert end-of-line markers */
len = convert_lineends(data, ptr, len); blen = convert_lineends(data, buf, blen);
} }
#endif #endif
/* it is one of those, at least */ /* it is one of those, at least */
@ -405,14 +409,14 @@ CURLcode Curl_client_write(struct Curl_easy *data,
/* INFO is only INFO */ /* INFO is only INFO */
DEBUGASSERT(!(type & CLIENTWRITE_INFO) || (type == CLIENTWRITE_INFO)); DEBUGASSERT(!(type & CLIENTWRITE_INFO) || (type == CLIENTWRITE_INFO));
if(type == CLIENTWRITE_BODY) { if(!data->req.writer_stack) {
if(data->req.ignorebody) result = do_init_stack(data);
return CURLE_OK; if(result)
return result;
if(data->req.writer_stack && !data->set.http_ce_skip) DEBUGASSERT(data->req.writer_stack);
return Curl_unencode_write(data, data->req.writer_stack, ptr, len);
} }
return chop_write(data, type, FALSE, ptr, len);
return Curl_cwriter_write(data, data->req.writer_stack, type, buf, blen);
} }
CURLcode Curl_client_unpause(struct Curl_easy *data) CURLcode Curl_client_unpause(struct Curl_easy *data)
@ -449,12 +453,12 @@ CURLcode Curl_client_unpause(struct Curl_easy *data)
void Curl_client_cleanup(struct Curl_easy *data) void Curl_client_cleanup(struct Curl_easy *data)
{ {
struct contenc_writer *writer = data->req.writer_stack; struct Curl_cwriter *writer = data->req.writer_stack;
size_t i; size_t i;
while(writer) { while(writer) {
data->req.writer_stack = writer->downstream; data->req.writer_stack = writer->next;
writer->handler->close_writer(data, writer); writer->cwt->do_close(data, writer);
free(writer); free(writer);
writer = data->req.writer_stack; writer = data->req.writer_stack;
} }
@ -466,58 +470,152 @@ void Curl_client_cleanup(struct Curl_easy *data)
} }
/* Real client writer: no downstream. */ /* Write data using an unencoding writer stack. "nbytes" is not
static CURLcode client_cew_init(struct Curl_easy *data, allowed to be 0. */
struct contenc_writer *writer) CURLcode Curl_cwriter_write(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes)
{ {
(void) data; if(!nbytes)
return CURLE_OK;
if(!writer)
return CURLE_WRITE_ERROR;
return writer->cwt->do_write(data, writer, type, buf, nbytes);
}
CURLcode Curl_cwriter_def_init(struct Curl_easy *data,
struct Curl_cwriter *writer)
{
(void)data;
(void)writer; (void)writer;
return CURLE_OK; return CURLE_OK;
} }
static CURLcode client_cew_write(struct Curl_easy *data, CURLcode Curl_cwriter_def_write(struct Curl_easy *data,
struct contenc_writer *writer, struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes) const char *buf, size_t nbytes)
{ {
(void)writer; return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!nbytes || data->req.ignorebody)
return CURLE_OK;
return chop_write(data, CLIENTWRITE_BODY, FALSE, (char *)buf, nbytes);
} }
static void client_cew_close(struct Curl_easy *data, void Curl_cwriter_def_close(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
(void) data; (void) data;
(void) writer; (void) writer;
} }
static const struct content_encoding client_cew = { /* Real client writer to installed callbacks. */
static CURLcode cw_client_write(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes)
{
(void)writer;
if(!nbytes)
return CURLE_OK;
return chop_write(data, type, FALSE, (char *)buf, nbytes);
}
static const struct Curl_cwtype cw_client = {
"client",
NULL, NULL,
Curl_cwriter_def_init,
cw_client_write,
Curl_cwriter_def_close,
sizeof(struct Curl_cwriter)
};
/* Download client writer in phase CURL_CW_PROTOCOL that
* sees the "real" download body data. */
static CURLcode cw_download_write(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes)
{
CURLcode result;
size_t nwrite;
if(!(type & CLIENTWRITE_BODY)) {
if((type & CLIENTWRITE_CONNECT) && data->set.suppress_connect_headers)
return CURLE_OK;
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
}
nwrite = nbytes;
data->req.bytecount += nbytes;
++data->req.bodywrites;
/* Enforce `max_filesize` also for downloads where we ignore the body.
* Also, write body data up to the max size. This ensures that we
* always produce the same result, even when buffers vary due to
* connection timings. test457 fails in CI randomly otherwise. */
if(data->set.max_filesize &&
(data->req.bytecount > data->set.max_filesize)) {
curl_off_t nexcess;
failf(data, "Exceeded the maximum allowed file size "
"(%" CURL_FORMAT_CURL_OFF_T ")",
data->set.max_filesize);
nexcess = data->req.bytecount - data->set.max_filesize;
nwrite = (nexcess >= (curl_off_t)nbytes)? 0 : (nbytes - (size_t)nexcess);
}
if(!data->req.ignorebody && nwrite) {
result = Curl_cwriter_write(data, writer->next, type, buf, nwrite);
if(result)
return result;
}
result = Curl_pgrsSetDownloadCounter(data, data->req.bytecount);
if(result)
return result;
return (nwrite == nbytes)? CURLE_OK : CURLE_FILESIZE_EXCEEDED;
}
static const struct Curl_cwtype cw_download = {
"download",
NULL, NULL,
client_cew_init, Curl_cwriter_def_init,
client_cew_write, cw_download_write,
client_cew_close, Curl_cwriter_def_close,
sizeof(struct contenc_writer) sizeof(struct Curl_cwriter)
};
/* RAW client writer in phase CURL_CW_RAW that
* enabled tracing of raw data. */
static CURLcode cw_raw_write(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes)
{
if(type & CLIENTWRITE_BODY && data->set.verbose && !data->req.ignorebody) {
Curl_debug(data, CURLINFO_DATA_IN, (char *)buf, nbytes);
}
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
}
static const struct Curl_cwtype cw_raw = {
"raw",
NULL,
Curl_cwriter_def_init,
cw_raw_write,
Curl_cwriter_def_close,
sizeof(struct Curl_cwriter)
}; };
/* Create an unencoding writer stage using the given handler. */ /* Create an unencoding writer stage using the given handler. */
CURLcode Curl_client_create_writer(struct contenc_writer **pwriter, CURLcode Curl_cwriter_create(struct Curl_cwriter **pwriter,
struct Curl_easy *data, struct Curl_easy *data,
const struct content_encoding *ce_handler, const struct Curl_cwtype *cwt,
int order) Curl_cwriter_phase phase)
{ {
struct contenc_writer *writer; struct Curl_cwriter *writer;
CURLcode result = CURLE_OUT_OF_MEMORY; CURLcode result = CURLE_OUT_OF_MEMORY;
DEBUGASSERT(ce_handler->writersize >= sizeof(struct contenc_writer)); DEBUGASSERT(cwt->cwriter_size >= sizeof(struct Curl_cwriter));
writer = (struct contenc_writer *) calloc(1, ce_handler->writersize); writer = (struct Curl_cwriter *) calloc(1, cwt->cwriter_size);
if(!writer) if(!writer)
goto out; goto out;
writer->handler = ce_handler; writer->cwt = cwt;
writer->order = order; writer->phase = phase;
result = ce_handler->init_writer(data, writer); result = cwt->do_init(data, writer);
out: out:
*pwriter = result? NULL : writer; *pwriter = result? NULL : writer;
@ -526,55 +624,74 @@ out:
return result; return result;
} }
void Curl_client_free_writer(struct Curl_easy *data, void Curl_cwriter_free(struct Curl_easy *data,
struct contenc_writer *writer) struct Curl_cwriter *writer)
{ {
if(writer) { if(writer) {
writer->handler->close_writer(data, writer); writer->cwt->do_close(data, writer);
free(writer); free(writer);
} }
} }
/* allow no more than 5 "chained" compression steps */ size_t Curl_cwriter_count(struct Curl_easy *data, Curl_cwriter_phase phase)
#define MAX_ENCODE_STACK 5
static CURLcode init_writer_stack(struct Curl_easy *data)
{ {
DEBUGASSERT(!data->req.writer_stack); struct Curl_cwriter *w;
return Curl_client_create_writer(&data->req.writer_stack, size_t n = 0;
data, &client_cew, 0);
for(w = data->req.writer_stack; w; w = w->next) {
if(w->phase == phase)
++n;
}
return n;
} }
CURLcode Curl_client_add_writer(struct Curl_easy *data, static CURLcode do_init_stack(struct Curl_easy *data)
struct contenc_writer *writer)
{ {
struct Curl_cwriter *writer;
CURLcode result; CURLcode result;
if(!data->req.writer_stack) { DEBUGASSERT(!data->req.writer_stack);
result = init_writer_stack(data); result = Curl_cwriter_create(&data->req.writer_stack,
data, &cw_client, CURL_CW_CLIENT);
if(result)
return result;
result = Curl_cwriter_create(&writer, data, &cw_download, CURL_CW_PROTOCOL);
if(result)
return result;
result = Curl_cwriter_add(data, writer);
if(result) {
Curl_cwriter_free(data, writer);
}
result = Curl_cwriter_create(&writer, data, &cw_raw, CURL_CW_RAW);
if(result)
return result;
result = Curl_cwriter_add(data, writer);
if(result) {
Curl_cwriter_free(data, writer);
}
return result;
}
CURLcode Curl_cwriter_add(struct Curl_easy *data,
struct Curl_cwriter *writer)
{
CURLcode result;
struct Curl_cwriter **anchor = &data->req.writer_stack;
if(!*anchor) {
result = do_init_stack(data);
if(result) if(result)
return result; return result;
} }
if(data->req.writer_stack_depth++ >= MAX_ENCODE_STACK) { /* Insert the writer as first in its phase.
failf(data, "Reject response due to more than %u content encodings", * Skip existing writers of lower phases. */
MAX_ENCODE_STACK); while(*anchor && (*anchor)->phase < writer->phase)
return CURLE_BAD_CONTENT_ENCODING; anchor = &((*anchor)->next);
} writer->next = *anchor;
*anchor = writer;
/* Stack the unencoding stage. */
if(writer->order >= data->req.writer_stack->order) {
writer->downstream = data->req.writer_stack;
data->req.writer_stack = writer;
}
else {
struct contenc_writer *w = data->req.writer_stack;
while(w->downstream && writer->order < w->downstream->order)
w = w->downstream;
writer->downstream = w->downstream;
w->downstream = writer;
}
return CURLE_OK; return CURLE_OK;
} }

127
lib/sendf.h
View File

@ -50,43 +50,122 @@
#define CLIENTWRITE_1XX (1<<5) /* a 1xx response related HEADER */ #define CLIENTWRITE_1XX (1<<5) /* a 1xx response related HEADER */
#define CLIENTWRITE_TRAILER (1<<6) /* a trailer HEADER */ #define CLIENTWRITE_TRAILER (1<<6) /* a trailer HEADER */
/**
* Write `len` bytes at `prt` to the client. `type` indicates what
* kind of data is being written.
*/
CURLcode Curl_client_write(struct Curl_easy *data, int type, char *ptr, CURLcode Curl_client_write(struct Curl_easy *data, int type, char *ptr,
size_t len) WARN_UNUSED_RESULT; size_t len) WARN_UNUSED_RESULT;
/**
* For a paused transfer, there might be buffered data held back.
* Attempt to flush this data to the client. This *may* trigger
* another pause of the transfer.
*/
CURLcode Curl_client_unpause(struct Curl_easy *data); CURLcode Curl_client_unpause(struct Curl_easy *data);
/**
* Free all resources related to client writing.
*/
void Curl_client_cleanup(struct Curl_easy *data); void Curl_client_cleanup(struct Curl_easy *data);
struct contenc_writer { /**
const struct content_encoding *handler; /* Encoding handler. */ * Client Writers - a chain passing transfer BODY data to the client.
struct contenc_writer *downstream; /* Downstream writer. */ * Main application: HTTP and related protocols
unsigned int order; /* Ordering within writer stack. */ * Other uses: monitoring of download progress
*
* Writers in the chain are order by their `phase`. First come all
* writers in CURL_CW_RAW, followed by any in CURL_CW_TRANSFER_DECODE,
* followed by any in CURL_CW_PROTOCOL, etc.
*
* When adding a writer, it is inserted as first in its phase. This means
* the order of adding writers of the same phase matters, but writers for
* different phases may be added in any order.
*
* Writers which do modify the BODY data written are expected to be of
* phases TRANSFER_DECODE or CONTENT_DECODE. The other phases are intended
* for monitoring writers. Which do *not* modify the data but gather
* statistics or update progress reporting.
*/
/* Phase a writer operates at. */
typedef enum {
CURL_CW_RAW, /* raw data written, before any decoding */
CURL_CW_TRANSFER_DECODE, /* remove transfer-encodings */
CURL_CW_PROTOCOL, /* after transfer, but before content decoding */
CURL_CW_CONTENT_DECODE, /* remove content-encodings */
CURL_CW_CLIENT /* data written to client */
} Curl_cwriter_phase;
/* Client Writer Type, provides the implementation */
struct Curl_cwtype {
const char *name; /* writer name. */
const char *alias; /* writer name alias, maybe NULL. */
CURLcode (*do_init)(struct Curl_easy *data,
struct Curl_cwriter *writer);
CURLcode (*do_write)(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes);
void (*do_close)(struct Curl_easy *data,
struct Curl_cwriter *writer);
size_t cwriter_size; /* sizeof() allocated struct Curl_cwriter */
}; };
/* Content encoding writer. */ /* Client writer instance */
struct content_encoding { struct Curl_cwriter {
const char *name; /* Encoding name. */ const struct Curl_cwtype *cwt; /* type implementation */
const char *alias; /* Encoding name alias. */ struct Curl_cwriter *next; /* Downstream writer. */
CURLcode (*init_writer)(struct Curl_easy *data, Curl_cwriter_phase phase; /* phase at which it operates */
struct contenc_writer *writer);
CURLcode (*unencode_write)(struct Curl_easy *data,
struct contenc_writer *writer,
const char *buf, size_t nbytes);
void (*close_writer)(struct Curl_easy *data,
struct contenc_writer *writer);
size_t writersize;
}; };
/**
* Create a new cwriter instance with given type and phase. Is not
* inserted into the writer chain by this call.
* Invokes `writer->do_init()`.
*/
CURLcode Curl_cwriter_create(struct Curl_cwriter **pwriter,
struct Curl_easy *data,
const struct Curl_cwtype *ce_handler,
Curl_cwriter_phase phase);
CURLcode Curl_client_create_writer(struct contenc_writer **pwriter, /**
struct Curl_easy *data, * Free a cwriter instance.
const struct content_encoding *ce_handler, * Invokes `writer->do_close()`.
int order); */
void Curl_cwriter_free(struct Curl_easy *data,
struct Curl_cwriter *writer);
void Curl_client_free_writer(struct Curl_easy *data, /**
struct contenc_writer *writer); * Count the number of writers installed of the given phase.
*/
size_t Curl_cwriter_count(struct Curl_easy *data, Curl_cwriter_phase phase);
CURLcode Curl_client_add_writer(struct Curl_easy *data, /**
struct contenc_writer *writer); * Adds a writer to the transfer's writer chain.
* The writers `phase` determines where in the chain it is inserted.
*/
CURLcode Curl_cwriter_add(struct Curl_easy *data,
struct Curl_cwriter *writer);
/**
* Convenience method for calling `writer->do_write()` that
* checks for NULL writer.
*/
CURLcode Curl_cwriter_write(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes);
/**
* Default implementations for do_init, do_write, do_close that
* do nothing and pass the data through.
*/
CURLcode Curl_cwriter_def_init(struct Curl_easy *data,
struct Curl_cwriter *writer);
CURLcode Curl_cwriter_def_write(struct Curl_easy *data,
struct Curl_cwriter *writer, int type,
const char *buf, size_t nbytes);
void Curl_cwriter_def_close(struct Curl_easy *data,
struct Curl_cwriter *writer);
/* internal read-function, does plain socket, SSL and krb4 */ /* internal read-function, does plain socket, SSL and krb4 */

7
lib/smb.c
View File

@ -1047,14 +1047,7 @@ static CURLcode smb_request_state(struct Curl_easy *data, bool *done)
break; break;
} }
} }
data->req.bytecount += len;
data->req.offset += len; data->req.offset += len;
result = Curl_pgrsSetDownloadCounter(data, data->req.bytecount);
if(result) {
req->result = result;
next_state = SMB_CLOSE;
break;
}
next_state = (len < MAX_PAYLOAD_SIZE) ? SMB_CLOSE : SMB_DOWNLOAD; next_state = (len < MAX_PAYLOAD_SIZE) ? SMB_CLOSE : SMB_DOWNLOAD;
break; break;

6
lib/tftp.c
View File

@ -1107,7 +1107,6 @@ static CURLcode tftp_receive_packet(struct Curl_easy *data)
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
struct connectdata *conn = data->conn; struct connectdata *conn = data->conn;
struct tftp_state_data *state = conn->proto.tftpc; struct tftp_state_data *state = conn->proto.tftpc;
struct SingleRequest *k = &data->req;
/* Receive the packet */ /* Receive the packet */
fromlen = sizeof(fromaddr); fromlen = sizeof(fromaddr);
@ -1141,11 +1140,6 @@ static CURLcode tftp_receive_packet(struct Curl_easy *data)
result = Curl_client_write(data, CLIENTWRITE_BODY, result = Curl_client_write(data, CLIENTWRITE_BODY,
(char *)state->rpacket.data + 4, (char *)state->rpacket.data + 4,
state->rbytes-4); state->rbytes-4);
if(!result) {
k->bytecount += state->rbytes-4;
result = Curl_pgrsSetDownloadCounter(data,
(curl_off_t) k->bytecount);
}
if(result) { if(result) {
tftp_state_machine(state, TFTP_EVENT_ERROR); tftp_state_machine(state, TFTP_EVENT_ERROR);
return result; return result;

53
lib/transfer.c
View File

@ -423,6 +423,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
{ {
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
ssize_t nread; /* number of bytes read */ ssize_t nread; /* number of bytes read */
ssize_t n_to_write;
bool readmore = FALSE; /* used by RTP to signal for more data */ bool readmore = FALSE; /* used by RTP to signal for more data */
int maxloops = 100; int maxloops = 100;
curl_off_t max_recv = data->set.max_recv_speed? curl_off_t max_recv = data->set.max_recv_speed?
@ -578,23 +579,6 @@ static CURLcode readwrite_data(struct Curl_easy *data,
} /* this is the first time we write a body part */ } /* this is the first time we write a body part */
#endif /* CURL_DISABLE_HTTP */ #endif /* CURL_DISABLE_HTTP */
k->bodywrites++;
/* pass data to the debug function before it gets "dechunked" */
if(data->set.verbose) {
if(k->badheader) {
Curl_debug(data, CURLINFO_DATA_IN,
Curl_dyn_ptr(&data->state.headerb),
Curl_dyn_len(&data->state.headerb));
if(k->badheader == HEADER_PARTHEADER)
Curl_debug(data, CURLINFO_DATA_IN,
k->str, (size_t)nread);
}
else
Curl_debug(data, CURLINFO_DATA_IN,
k->str, (size_t)nread);
}
#ifndef CURL_DISABLE_HTTP #ifndef CURL_DISABLE_HTTP
if(k->chunk) { if(k->chunk) {
/* /*
@ -634,16 +618,26 @@ static CURLcode readwrite_data(struct Curl_easy *data,
#endif /* CURL_DISABLE_HTTP */ #endif /* CURL_DISABLE_HTTP */
/* Account for body content stored in the header buffer */ /* Account for body content stored in the header buffer */
n_to_write = nread;
if((k->badheader == HEADER_PARTHEADER) && !k->ignorebody) { if((k->badheader == HEADER_PARTHEADER) && !k->ignorebody) {
size_t headlen = Curl_dyn_len(&data->state.headerb); n_to_write += Curl_dyn_len(&data->state.headerb);
DEBUGF(infof(data, "Increasing bytecount by %zu", headlen));
k->bytecount += headlen;
} }
if((-1 != k->maxdownload) && if((-1 != k->maxdownload) &&
(k->bytecount + nread >= k->maxdownload)) { (k->bytecount + n_to_write >= k->maxdownload)) {
excess = (size_t)(k->bytecount + n_to_write - k->maxdownload);
if(excess > 0 && !k->ignorebody) {
infof(data,
"Excess found in a read:"
" excess = %zu"
", size = %" CURL_FORMAT_CURL_OFF_T
", maxdownload = %" CURL_FORMAT_CURL_OFF_T
", bytecount = %" CURL_FORMAT_CURL_OFF_T,
excess, k->size, k->maxdownload, k->bytecount);
connclose(conn, "excess found in a read");
}
excess = (size_t)(k->bytecount + nread - k->maxdownload);
nread = (ssize_t) (k->maxdownload - k->bytecount); nread = (ssize_t) (k->maxdownload - k->bytecount);
if(nread < 0) /* this should be unusual */ if(nread < 0) /* this should be unusual */
nread = 0; nread = 0;
@ -657,17 +651,12 @@ static CURLcode readwrite_data(struct Curl_easy *data,
} }
} }
k->bytecount += nread;
max_recv -= nread; max_recv -= nread;
result = Curl_pgrsSetDownloadCounter(data, k->bytecount);
if(result)
goto out;
if(!k->chunk && (nread || k->badheader || is_empty_data)) { if(!k->chunk && (nread || k->badheader || is_empty_data)) {
/* If this is chunky transfer, it was already written */ /* If this is chunky transfer, it was already written */
if(k->badheader && !k->ignorebody) { if(k->badheader) {
/* we parsed a piece of data wrongly assuming it was a header /* we parsed a piece of data wrongly assuming it was a header
and now we output it as body instead */ and now we output it as body instead */
size_t headlen = Curl_dyn_len(&data->state.headerb); size_t headlen = Curl_dyn_len(&data->state.headerb);
@ -691,10 +680,12 @@ static CURLcode readwrite_data(struct Curl_easy *data,
in http_chunks.c. in http_chunks.c.
Make sure that ALL_CONTENT_ENCODINGS contains all the Make sure that ALL_CONTENT_ENCODINGS contains all the
encodings handled here. */ encodings handled here. */
if(!k->ignorebody && nread) { if(nread) {
#ifndef CURL_DISABLE_POP3 #ifndef CURL_DISABLE_POP3
if(conn->handler->protocol & PROTO_FAMILY_POP3) if(conn->handler->protocol & PROTO_FAMILY_POP3) {
result = Curl_pop3_write(data, k->str, nread); result = k->ignorebody? CURLE_OK :
Curl_pop3_write(data, k->str, nread);
}
else else
#endif /* CURL_DISABLE_POP3 */ #endif /* CURL_DISABLE_POP3 */
result = Curl_client_write(data, CLIENTWRITE_BODY, k->str, result = Curl_client_write(data, CLIENTWRITE_BODY, k->str,

2
lib/urldata.h
View File

@ -685,7 +685,7 @@ struct SingleRequest {
enum upgrade101 upgr101; /* 101 upgrade state */ enum upgrade101 upgr101; /* 101 upgrade state */
/* Content unencoding stack. See sec 3.5, RFC2616. */ /* Content unencoding stack. See sec 3.5, RFC2616. */
struct contenc_writer *writer_stack; struct Curl_cwriter *writer_stack;
time_t timeofdoc; time_t timeofdoc;
long bodywrites; long bodywrites;
char *location; /* This points to an allocated version of the Location: char *location; /* This points to an allocated version of the Location:

12
lib/vssh/libssh.c
View File

@ -1466,13 +1466,7 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
state(data, SSH_STOP); state(data, SSH_STOP);
break; break;
} }
/* since this counts what we send to the client, we include the
newline in this counter */
data->req.bytecount += sshc->readdir_len + 1;
/* output debug output if that is requested */
Curl_debug(data, CURLINFO_DATA_OUT, (char *)sshc->readdir_filename,
sshc->readdir_len);
} }
else { else {
if(Curl_dyn_add(&sshc->readdir_buf, sshc->readdir_longentry)) { if(Curl_dyn_add(&sshc->readdir_buf, sshc->readdir_longentry)) {
@ -1564,12 +1558,6 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
Curl_dyn_ptr(&sshc->readdir_buf), Curl_dyn_ptr(&sshc->readdir_buf),
Curl_dyn_len(&sshc->readdir_buf)); Curl_dyn_len(&sshc->readdir_buf));
if(!result) {
/* output debug output if that is requested */
Curl_debug(data, CURLINFO_DATA_OUT, Curl_dyn_ptr(&sshc->readdir_buf),
Curl_dyn_len(&sshc->readdir_buf));
data->req.bytecount += Curl_dyn_len(&sshc->readdir_buf);
}
ssh_string_free_char(sshc->readdir_tmp); ssh_string_free_char(sshc->readdir_tmp);
sshc->readdir_tmp = NULL; sshc->readdir_tmp = NULL;

14
lib/vssh/libssh2.c
View File

@ -2341,14 +2341,7 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
state(data, SSH_STOP); state(data, SSH_STOP);
break; break;
} }
/* since this counts what we send to the client, we include the
newline in this counter */
data->req.bytecount += readdir_len + 1;
/* output debug output if that is requested */
Curl_debug(data, CURLINFO_DATA_IN, sshp->readdir_filename,
readdir_len);
Curl_debug(data, CURLINFO_DATA_IN, (char *)"\n", 1);
} }
else { else {
result = Curl_dyn_add(&sshp->readdir, sshp->readdir_longentry); result = Curl_dyn_add(&sshp->readdir, sshp->readdir_longentry);
@ -2427,13 +2420,6 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
Curl_dyn_ptr(&sshp->readdir), Curl_dyn_ptr(&sshp->readdir),
Curl_dyn_len(&sshp->readdir)); Curl_dyn_len(&sshp->readdir));
if(!result) {
/* output debug output if that is requested */
Curl_debug(data, CURLINFO_DATA_IN,
Curl_dyn_ptr(&sshp->readdir),
Curl_dyn_len(&sshp->readdir));
data->req.bytecount += Curl_dyn_len(&sshp->readdir);
}
if(result) { if(result) {
Curl_dyn_free(&sshp->readdir); Curl_dyn_free(&sshp->readdir);
state(data, SSH_STOP); state(data, SSH_STOP);

5
tests/data/test457
View File

@ -20,7 +20,8 @@ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
30 30
bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
21;heresatest=moooo 21;heresatest=moooo
cccccccccccccccccccccccccccccccc cccccccccccccccccccccccccccccc
c
0 0
@ -31,7 +32,7 @@ Server: fakeit/0.9 fakeitbad/1.0
Transfer-Encoding: chunked Transfer-Encoding: chunked
Connection: mooo Connection: mooo
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccc
</datacheck> </datacheck>
</reply> </reply>