http: write last header line late

- HEADERFUNCTIONS might inspect response properties like
  CURLINFO_CONTENT_LENGTH_DOWNLOAD_T on seeing the last header line. If
  the line is being written before this is initialized, values are not
  available.

- write the last header line late when analyzing a HTTP response so that
  all information is available at the time of the writing.

- add test1485 to verify that CURLINFO_CONTENT_LENGTH_DOWNLOAD_T works
  on seeing the last header.

Fixes #13752
Reported-by: Harry Sintonen
Closes #13757

lib/http.c

@@ -3363,8 +3363,35 @@ CURLcode Curl_bump_headersize(struct Curl_easy *data,
   return CURLE_OK;
 }
 
+static CURLcode http_write_header(struct Curl_easy *data,
+                                  const char *hd, size_t hdlen)
+{
+  CURLcode result;
+  int writetype;
+
+  /* now, only output this if the header AND body are requested:
+   */
+  Curl_debug(data, CURLINFO_HEADER_IN, (char *)hd, hdlen);
+
+  writetype = CLIENTWRITE_HEADER |
+    ((data->req.httpcode/100 == 1) ? CLIENTWRITE_1XX : 0);
+
+  result = Curl_client_write(data, writetype, hd, hdlen);
+  if(result)
+    return result;
+
+  result = Curl_bump_headersize(data, hdlen, FALSE);
+  if(result)
+    return result;
+
+  data->req.deductheadercount = (100 <= data->req.httpcode &&
+                                 199 >= data->req.httpcode)?
+                                data->req.headerbytecount:0;
+  return result;
+}
 
 static CURLcode http_on_response(struct Curl_easy *data,
+                                 const char *last_hd, size_t last_hd_len,
                                  const char *buf, size_t blen,
                                  size_t *pconsumed)
 {
@@ -3384,9 +3411,20 @@ static CURLcode http_on_response(struct Curl_easy *data,
     conn->bundle->multiuse = BUNDLE_NO_MULTIUSE;
   }
 
+  if(k->httpcode < 200 && last_hd) {
+    /* Intermediate responses might trigger processing of more
+     * responses, write the last header to the client before
+     * proceeding. */
+    result = http_write_header(data, last_hd, last_hd_len);
+    last_hd = NULL; /* handled it */
+    if(result)
+      goto out;
+  }
+
   if(k->httpcode < 100) {
     failf(data, "Unsupported response code in HTTP response");
-    return CURLE_UNSUPPORTED_PROTOCOL;
+    result = CURLE_UNSUPPORTED_PROTOCOL;
+    goto out;
   }
   else if(k->httpcode < 200) {
     /* "A user agent MAY ignore unexpected 1xx status responses."
@@ -3405,10 +3443,12 @@ static CURLcode http_on_response(struct Curl_easy *data,
       break;
     case 101:
       /* Switching Protocols only allowed from HTTP/1.1 */
+
       if(conn->httpversion != 11) {
         /* invalid for other HTTP versions */
         failf(data, "unexpected 101 response code");
-        return CURLE_WEIRD_SERVER_REPLY;
+        result = CURLE_WEIRD_SERVER_REPLY;
+        goto out;
       }
       if(k->upgr101 == UPGR101_H2) {
         /* Switching to HTTP/2, where we will get more responses */
@@ -3421,7 +3461,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
          * be processed. */
         result = Curl_http2_upgrade(data, conn, FIRSTSOCKET, buf, blen);
         if(result)
-          return result;
+          goto out;
         *pconsumed += blen;
       }
 #ifdef USE_WEBSOCKETS
@@ -3430,7 +3470,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
          * WebSockets format and taken in by the protocol handler. */
         result = Curl_ws_accept(data, buf, blen);
         if(result)
-          return result;
+          goto out;
         *pconsumed += blen; /* ws accept handled the data */
         k->header = FALSE; /* we will not get more responses */
         if(data->set.connect_only)
@@ -3451,7 +3491,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
        * to receive a final response eventually. */
       break;
     }
-    return result;
+    goto out;
   }
 
   /* k->httpcode >= 200, final response */
@@ -3512,7 +3552,8 @@ static CURLcode http_on_response(struct Curl_easy *data,
   /* All >=200 HTTP status codes are errors when wanting websockets */
   if(data->req.upgr101 == UPGR101_WS) {
     failf(data, "Refused WebSockets upgrade: %d", k->httpcode);
-    return CURLE_HTTP_RETURNED_ERROR;
+    result = CURLE_HTTP_RETURNED_ERROR;
+    goto out;
   }
 #endif
 
@@ -3520,7 +3561,8 @@ static CURLcode http_on_response(struct Curl_easy *data,
   if(http_should_fail(data, data->req.httpcode)) {
     failf(data, "The requested URL returned error: %d",
           k->httpcode);
-    return CURLE_HTTP_RETURNED_ERROR;
+    result = CURLE_HTTP_RETURNED_ERROR;
+    goto out;
   }
 
   /* Curl_http_auth_act() checks what authentication methods
@@ -3528,7 +3570,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
    * use. It will set 'newurl' if an auth method was picked. */
   result = Curl_http_auth_act(data);
   if(result)
-    return result;
+    goto out;
 
   if(k->httpcode >= 300) {
     if((!data->req.authneg) && !conn->bits.close &&
@@ -3566,7 +3608,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
                           "Stop sending data before everything sent");
               result = http_perhapsrewind(data, conn);
               if(result)
-                return result;
+                goto out;
             }
             data->state.disableexpect = TRUE;
             DEBUGASSERT(!data->req.newurl);
@@ -3582,7 +3624,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
             streamclose(conn, "Stop sending data before everything sent");
             result = Curl_req_abort_sending(data);
             if(result)
-              return result;
+              goto out;
           }
         }
         break;
@@ -3605,7 +3647,7 @@ static CURLcode http_on_response(struct Curl_easy *data,
    */
   result = Curl_http_size(data);
   if(result)
-    return result;
+    goto out;
 
   /* If we requested a "no body", this is a good time to get
    * out and return home.
@@ -3624,7 +3666,16 @@ static CURLcode http_on_response(struct Curl_easy *data,
     k->download_done = TRUE;
 
   /* final response without error, prepare to receive the body */
-  return Curl_http_firstwrite(data);
+  result = Curl_http_firstwrite(data);
+
+out:
+  if(last_hd) {
+    /* if not written yet, write it now */
+    CURLcode r2 = http_write_header(data, last_hd, last_hd_len);
+    if(!result)
+      result = r2;
+  }
+  return result;
 }
 
 static CURLcode http_rw_hd(struct Curl_easy *data,
@@ -3639,36 +3690,25 @@ static CURLcode http_rw_hd(struct Curl_easy *data,
   *pconsumed = 0;
   if((0x0a == *hd) || (0x0d == *hd)) {
     /* Empty header line means end of headers! */
+    struct dynbuf last_header;
     size_t consumed;
 
-    /* now, only output this if the header AND body are requested:
+    Curl_dyn_init(&last_header, hdlen + 1);
-     */
+    result = Curl_dyn_addn(&last_header, hd, hdlen);
-    Curl_debug(data, CURLINFO_HEADER_IN, (char *)hd, hdlen);
-
-    writetype = CLIENTWRITE_HEADER |
-      ((k->httpcode/100 == 1) ? CLIENTWRITE_1XX : 0);
-
-    result = Curl_client_write(data, writetype, hd, hdlen);
     if(result)
       return result;
 
-    result = Curl_bump_headersize(data, hdlen, FALSE);
-    if(result)
-      return result;
-
-    data->req.deductheadercount =
-      (100 <= k->httpcode && 199 >= k->httpcode)?data->req.headerbytecount:0;
-
     /* analyze the response to find out what to do. */
     /* Caveat: we clear anything in the header brigade, because a
      * response might switch HTTP version which may call use recursively.
      * Not nice, but that is currently the way of things. */
     Curl_dyn_reset(&data->state.headerb);
-    result = http_on_response(data, buf_remain, blen, &consumed);
+    result = http_on_response(data, Curl_dyn_ptr(&last_header),
-    if(result)
+                              Curl_dyn_len(&last_header),
-      return result;
+                              buf_remain, blen, &consumed);
     *pconsumed += consumed;
-    return CURLE_OK;
+    Curl_dyn_free(&last_header);
+    return result;
   }
 
   /*

tests/data/Makefile.inc

@@ -188,7 +188,7 @@ test1447 test1448 test1449 test1450 test1451 test1452 test1453 test1454 \
 test1455 test1456 test1457 test1458 test1459 test1460 test1461 test1462 \
 test1463 test1464 test1465 test1466 test1467 test1468 test1469 test1470 \
 test1471 test1472 test1473 test1474 test1475 test1476 test1477 test1478 \
-test1479 test1480 test1481 test1482 test1483 test1484 \
+test1479 test1480 test1481 test1482 test1483 test1484 test1485 \
 \
 test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \
 test1508 test1509 test1510 test1511 test1512 test1513 test1514 test1515 \

tests/data/test1485

@@ -0,0 +1,51 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK
+Server: Someone
+Content-Length: 7
+
+123456
+</data>
+<datacheck>
+HTTP/1.1 200 OK
+Server: Someone
+Content-Length: 7
+
+123456
+</datacheck>
+</reply>
+# Client-side
+<client>
+<server>
+http
+</server>
+<tool>
+lib%TESTNUMBER
+</tool>
+<name>
+get curlinfo on last header in callback
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/libtest/Makefile.inc

@@ -54,6 +54,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect libprereq      \
  lib670 lib671 lib672 lib673 lib674 lib676 lib677 lib678 \
  lib1156 \
  lib1301 \
+ lib1485 \
  lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 \
  lib1509 lib1510 lib1511 lib1512 lib1513 lib1514 lib1515         lib1517 \
  lib1518         lib1520 lib1521 lib1522 lib1523 \
@@ -346,6 +347,9 @@ lib678_LDADD = $(TESTUTIL_LIBS)
 lib1301_SOURCES = lib1301.c $(SUPPORTFILES) $(TESTUTIL)
 lib1301_LDADD = $(TESTUTIL_LIBS)
 
+lib1485_SOURCES = lib1485.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib1485_LDADD = $(TESTUTIL_LIBS)
+
 lib1500_SOURCES = lib1500.c $(SUPPORTFILES) $(TESTUTIL)
 lib1500_LDADD = $(TESTUTIL_LIBS)
 

tests/libtest/lib1485.c

@@ -0,0 +1,120 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * SPDX-License-Identifier: curl
+ *
+ ***************************************************************************/
+#include "test.h"
+
+#include "testutil.h"
+#include "warnless.h"
+#include "memdebug.h"
+
+struct transfer_status {
+  CURL *easy;
+  curl_off_t out_len;
+  size_t hd_line;
+  CURLcode result;
+  int http_status;
+};
+
+static size_t header_callback(void *ptr, size_t size, size_t nmemb,
+                              void *userp)
+{
+  struct transfer_status *st = (struct transfer_status *)userp;
+  const char *hd = ptr;
+  size_t len = size * nmemb;
+  CURLcode result;
+
+  (void)fwrite(ptr, size, nmemb, stdout);
+  ++st->hd_line;
+  if(len == 2 && hd[0] == '\r' && hd[1] == '\n') {
+    curl_off_t clen;
+    long httpcode = 0;
+    /* end of a response */
+    result = curl_easy_getinfo(st->easy, CURLINFO_RESPONSE_CODE, &httpcode);
+    fprintf(stderr, "header_callback, get status: %ld, %d\n",
+            httpcode, result);
+    if(httpcode < 100 || httpcode >= 1000) {
+      fprintf(stderr, "header_callback, invalid status: %ld, %d\n",
+              httpcode, result);
+      return CURLE_WRITE_ERROR;
+    }
+    st->http_status = (int)httpcode;
+    if(st->http_status >= 200 && st->http_status < 300) {
+      result = curl_easy_getinfo(st->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T,
+                                 &clen);
+      fprintf(stderr, "header_callback, info Content-Length: %ld, %d\n",
+              (long)clen, result);
+      if(result) {
+        st->result = result;
+        return CURLE_WRITE_ERROR;
+      }
+      if(clen < 0) {
+        fprintf(stderr, "header_callback, expected known Content-Length, "
+                "got: %ld\n", (long)clen);
+        return CURLE_WRITE_ERROR;
+      }
+    }
+  }
+  return len;
+}
+
+static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp)
+{
+  struct transfer_status *st = (struct transfer_status *)userp;
+  size_t len = size * nmemb;
+  fwrite(ptr, size, nmemb, stdout);
+  st->out_len += (curl_off_t)len;
+  return len;
+}
+
+CURLcode test(char *URL)
+{
+  CURL *curls = NULL;
+  CURLcode res = CURLE_OK;
+  struct transfer_status st;
+
+  start_test_timing();
+
+  memset(&st, 0, sizeof(st));
+
+  global_init(CURL_GLOBAL_ALL);
+
+  easy_init(curls);
+  st.easy = curls; /* to allow callbacks access */
+
+  easy_setopt(curls, CURLOPT_URL, URL);
+  easy_setopt(curls, CURLOPT_WRITEFUNCTION, write_callback);
+  easy_setopt(curls, CURLOPT_WRITEDATA, &st);
+  easy_setopt(curls, CURLOPT_HEADERFUNCTION, header_callback);
+  easy_setopt(curls, CURLOPT_HEADERDATA, &st);
+
+  easy_setopt(curls, CURLOPT_NOPROGRESS, 1L);
+
+  res = curl_easy_perform(curls);
+
+test_cleanup:
+
+  curl_easy_cleanup(curls);
+  curl_global_cleanup();
+
+  return res; /* return the final return code */
+}