lib: xfer_setup and non-blocking shutdown

- clarify Curl_xfer_setup() with RECV/SEND flags and different calls for
  which socket they operate on. Add a shutdown flag for secondary
  sockets
- change Curl_xfer_setup() calls to new functions
- implement non-blocking connection shutdown at the end of receiving or
  sending a transfer

Closes #13913

lib/c-hyper.c

@@ -1133,7 +1133,7 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
     Curl_pgrsSetUploadSize(data, 0); /* nothing */
   }
 
-  Curl_xfer_setup(data, FIRSTSOCKET, -1, TRUE, FIRSTSOCKET);
+  Curl_xfer_setup1(data, CURL_XFER_SENDRECV, -1, TRUE);
   conn->datastream = Curl_hyper_stream;
 
   /* clear userpwd and proxyuserpwd to avoid reusing old credentials

lib/cf-socket.c

@@ -1028,7 +1028,6 @@ static CURLcode cf_socket_shutdown(struct Curl_cfilter *cf,
       unsigned char buf[1024];
       (void)sread(ctx->sock, buf, sizeof(buf));
     }
-    cf_socket_close(cf, data);
   }
   *done = TRUE;
   return CURLE_OK;

lib/cfilters.c

@@ -175,60 +175,54 @@ void Curl_conn_close(struct Curl_easy *data, int index)
   if(cf) {
     cf->cft->do_close(cf, data);
   }
+  Curl_shutdown_clear(data, index);
 }
 
-CURLcode Curl_conn_shutdown_blocking(struct Curl_easy *data, int sockindex)
+CURLcode Curl_conn_shutdown(struct Curl_easy *data, int sockindex, bool *done)
 {
   struct Curl_cfilter *cf;
   CURLcode result = CURLE_OK;
+  timediff_t timeout_ms;
+  struct curltime now;
 
   DEBUGASSERT(data->conn);
   /* it is valid to call that without filters being present */
   cf = data->conn->cfilter[sockindex];
-  if(cf) {
+  if(!cf) {
-    timediff_t timeout_ms;
+    *done = TRUE;
-    bool done = FALSE;
+    return CURLE_OK;
-    int what;
+  }
 
+  *done = FALSE;
+  now = Curl_now();
+  if(!Curl_shutdown_started(data, sockindex)) {
     DEBUGF(infof(data, "shutdown start on%s connection",
            sockindex? " secondary" : ""));
-    Curl_shutdown_start(data, sockindex, NULL);
+    Curl_shutdown_start(data, sockindex, &now);
-    while(cf) {
-      while(!done && !result) {
-        result = cf->cft->do_shutdown(cf, data, &done);
-        if(!result && !done) {
-          timeout_ms = Curl_shutdown_timeleft(data->conn, sockindex, NULL);
-          if(timeout_ms < 0) {
-            failf(data, "SSL shutdown timeout");
-            result = CURLE_OPERATION_TIMEDOUT;
-            goto out;
-          }
-
-          what = Curl_conn_cf_poll(cf, data, timeout_ms);
-          if(what < 0) {
-            /* fatal error */
-            failf(data, "select/poll on SSL socket, errno: %d", SOCKERRNO);
-            result = CURLE_RECV_ERROR;
-            goto out;
-          }
-          else if(0 == what) {
-            failf(data, "SSL shutdown timeout");
-            result = CURLE_OPERATION_TIMEDOUT;
-            goto out;
-          }
-        }
-      }
-      if(result)
-        break;
-      CURL_TRC_CF(data, cf, "shut down successfully");
-      cf = cf->next;
-      done = FALSE;
-    }
-    Curl_shutdown_clear(data, sockindex);
-    DEBUGF(infof(data, "shutdown done on%s connection -> %d",
-           sockindex? " secondary" : "", result));
   }
-out:
+  else {
+    timeout_ms = Curl_shutdown_timeleft(data->conn, sockindex, &now);
+    if(timeout_ms < 0) {
+      failf(data, "SSL shutdown timeout");
+      return CURLE_OPERATION_TIMEDOUT;
+    }
+  }
+
+  while(cf) {
+    bool cfdone = FALSE;
+    result = cf->cft->do_shutdown(cf, data, &cfdone);
+    if(result) {
+      CURL_TRC_CF(data, cf, "shut down failed with %d", result);
+      return result;
+    }
+    else if(!cfdone) {
+      CURL_TRC_CF(data, cf, "shut down not done yet");
+      return CURLE_OK;
+    }
+    CURL_TRC_CF(data, cf, "shut down successfully");
+    cf = cf->next;
+  }
+  *done = (!result);
   return result;
 }
 

lib/cfilters.h

@@ -384,10 +384,11 @@ bool Curl_conn_is_multiplex(struct connectdata *conn, int sockindex);
 void Curl_conn_close(struct Curl_easy *data, int sockindex);
 
 /**
- * Shutdown the connection at `sockindex` blocking with timeout
+ * Shutdown the connection at `sockindex` non-blocking, using timeout
- * from `data->set.shutdowntimeout`, default DEFAULT_SHUTDOWN_TIMEOUT_MS
+ * from `data->set.shutdowntimeout`, default DEFAULT_SHUTDOWN_TIMEOUT_MS.
+ * Will return CURLE_OK and *done == FALSE if not finished.
  */
-CURLcode Curl_conn_shutdown_blocking(struct Curl_easy *data, int sockindex);
+CURLcode Curl_conn_shutdown(struct Curl_easy *data, int sockindex, bool *done);
 
 /**
  * Return if data is pending in some connection filter at chain

lib/connect.c

@@ -179,6 +179,12 @@ void Curl_shutdown_clear(struct Curl_easy *data, int sockindex)
   memset(pt, 0, sizeof(*pt));
 }
 
+bool Curl_shutdown_started(struct Curl_easy *data, int sockindex)
+{
+  struct curltime *pt = &data->conn->shutdown.start[sockindex];
+  return (pt->tv_sec > 0) || (pt->tv_usec > 0);
+}
+
 /* Copies connection info into the transfer handle to make it available when
    the transfer handle is no longer associated with the connection. */
 void Curl_persistconninfo(struct Curl_easy *data, struct connectdata *conn,

lib/connect.h

@@ -52,6 +52,9 @@ timediff_t Curl_shutdown_timeleft(struct connectdata *conn, int sockindex,
 
 void Curl_shutdown_clear(struct Curl_easy *data, int sockindex);
 
+/* TRUE iff shutdown has been started */
+bool Curl_shutdown_started(struct Curl_easy *data, int sockindex);
+
 /*
  * Used to extract socket and connectdata struct for the most recent
  * transfer on the given Curl_easy.

lib/curl_rtmp.c

@@ -273,10 +273,10 @@ static CURLcode rtmp_do(struct Curl_easy *data, bool *done)
 
   if(data->state.upload) {
     Curl_pgrsSetUploadSize(data, data->state.infilesize);
-    Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+    Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
   }
   else
-    Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1);
+    Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE);
   *done = TRUE;
   return CURLE_OK;
 }

lib/dict.c

@@ -241,7 +241,7 @@ static CURLcode dict_do(struct Curl_easy *data, bool *done)
       failf(data, "Failed sending DICT request");
       goto error;
     }
-    Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1); /* no upload */
+    Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE); /* no upload */
   }
   else if(strncasecompare(path, DICT_DEFINE, sizeof(DICT_DEFINE)-1) ||
           strncasecompare(path, DICT_DEFINE2, sizeof(DICT_DEFINE2)-1) ||
@@ -287,7 +287,7 @@ static CURLcode dict_do(struct Curl_easy *data, bool *done)
       failf(data, "Failed sending DICT request");
       goto error;
     }
-    Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1);
+    Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE);
   }
   else {
 
@@ -309,7 +309,7 @@ static CURLcode dict_do(struct Curl_easy *data, bool *done)
         goto error;
       }
 
-      Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1);
+      Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE);
     }
   }
 

lib/ftp.c

@@ -290,12 +290,8 @@ const struct Curl_handler Curl_handler_ftps = {
 };
 #endif
 
-static void close_secondarysocket(struct Curl_easy *data, bool premature)
+static void close_secondarysocket(struct Curl_easy *data)
 {
-  if(!premature) {
-    CURL_TRC_FTP(data, "[%s] shutting down DATA connection", FTP_DSTATE(data));
-    Curl_conn_shutdown_blocking(data, SECONDARYSOCKET);
-  }
   CURL_TRC_FTP(data, "[%s] closing DATA connection", FTP_DSTATE(data));
   Curl_conn_close(data, SECONDARYSOCKET);
   Curl_conn_cf_discard_all(data, data->conn, SECONDARYSOCKET);
@@ -478,7 +474,7 @@ static CURLcode AcceptServerConnect(struct Curl_easy *data)
     Curl_set_in_callback(data, false);
 
     if(error) {
-      close_secondarysocket(data, TRUE);
+      close_secondarysocket(data);
       return CURLE_ABORTED_BY_CALLBACK;
     }
   }
@@ -659,12 +655,12 @@ static CURLcode InitiateTransfer(struct Curl_easy *data)
     /* set the SO_SNDBUF for the secondary socket for those who need it */
     Curl_sndbuf_init(conn->sock[SECONDARYSOCKET]);
 
-    Curl_xfer_setup(data, -1, -1, FALSE, SECONDARYSOCKET);
+    Curl_xfer_setup2(data, CURL_XFER_SEND, -1, TRUE);
   }
   else {
     /* FTP download: */
-    Curl_xfer_setup(data, SECONDARYSOCKET,
+    Curl_xfer_setup2(data, CURL_XFER_RECV,
-                    conn->proto.ftpc.retr_size_saved, FALSE, -1);
+                     conn->proto.ftpc.retr_size_saved, TRUE);
   }
 
   conn->proto.ftpc.pp.pending_resp = TRUE; /* expect server response */
@@ -1739,7 +1735,7 @@ static CURLcode ftp_state_ul_setup(struct Curl_easy *data,
         infof(data, "File already completely uploaded");
 
         /* no data to transfer */
-        Curl_xfer_setup(data, -1, -1, FALSE, -1);
+        Curl_xfer_setup_nop(data);
 
         /* Set ->transfer so that we won't get any error in
          * ftp_done() because we didn't transfer anything! */
@@ -2410,7 +2406,7 @@ static CURLcode ftp_state_retr(struct Curl_easy *data,
 
     if(ftp->downloadsize == 0) {
       /* no data to transfer */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
       infof(data, "File already completely downloaded");
 
       /* Set ->transfer so that we won't get any error in ftp_done()
@@ -3466,7 +3462,7 @@ static CURLcode ftp_done(struct Curl_easy *data, CURLcode status,
       }
     }
 
-    close_secondarysocket(data, result != CURLE_OK);
+    close_secondarysocket(data);
   }
 
   if(!result && (ftp->transfer == PPTRANSFER_BODY) && ftpc->ctl_valid &&
@@ -3833,7 +3829,7 @@ static CURLcode ftp_do_more(struct Curl_easy *data, int *completep)
   }
 
   /* no data to transfer */
-  Curl_xfer_setup(data, -1, -1, FALSE, -1);
+  Curl_xfer_setup_nop(data);
 
   if(!ftpc->wait_data_conn) {
     /* no waiting for the data connection so this is now complete */
@@ -4434,14 +4430,14 @@ static CURLcode ftp_dophase_done(struct Curl_easy *data, bool connected)
     CURLcode result = ftp_do_more(data, &completed);
 
     if(result) {
-      close_secondarysocket(data, TRUE);
+      close_secondarysocket(data);
       return result;
     }
   }
 
   if(ftp->transfer != PPTRANSFER_BODY)
     /* no data to transfer */
-    Curl_xfer_setup(data, -1, -1, FALSE, -1);
+    Curl_xfer_setup_nop(data);
   else if(!connected)
     /* since we didn't connect now, we want do_more to get called */
     conn->bits.do_more = TRUE;

lib/gopher.c

@@ -238,7 +238,7 @@ static CURLcode gopher_do(struct Curl_easy *data, bool *done)
   if(result)
     return result;
 
-  Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1);
+  Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE);
   return CURLE_OK;
 }
 #endif /* CURL_DISABLE_GOPHER */

lib/http.c

@@ -2247,7 +2247,7 @@ CURLcode Curl_http_req_complete(struct Curl_easy *data,
 out:
   if(!result) {
     /* setup variables for the upcoming transfer */
-    Curl_xfer_setup(data, FIRSTSOCKET, -1, TRUE, FIRSTSOCKET);
+    Curl_xfer_setup1(data, CURL_XFER_SENDRECV, -1, TRUE);
   }
   return result;
 }

lib/imap.c

@@ -1214,14 +1214,14 @@ static CURLcode imap_state_fetch_resp(struct Curl_easy *data,
 
     if(data->req.bytecount == size)
       /* The entire data is already transferred! */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
     else {
       /* IMAP download */
       data->req.maxdownload = size;
       /* force a recv/send check of this connection, as the data might've been
        read off the socket already */
       data->state.select_bits = CURL_CSELECT_IN;
-      Curl_xfer_setup(data, FIRSTSOCKET, size, FALSE, -1);
+      Curl_xfer_setup1(data, CURL_XFER_RECV, size, FALSE);
     }
   }
   else {
@@ -1269,7 +1269,7 @@ static CURLcode imap_state_append_resp(struct Curl_easy *data, int imapcode,
     Curl_pgrsSetUploadSize(data, data->state.infilesize);
 
     /* IMAP upload */
-    Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+    Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
     /* End of DO phase */
     imap_state(data, IMAP_STOP);
@@ -1694,7 +1694,7 @@ static CURLcode imap_dophase_done(struct Curl_easy *data, bool connected)
 
   if(imap->transfer != PPTRANSFER_BODY)
     /* no data to transfer */
-    Curl_xfer_setup(data, -1, -1, FALSE, -1);
+    Curl_xfer_setup_nop(data);
 
   return CURLE_OK;
 }

lib/ldap.c

@@ -758,7 +758,7 @@ quit:
   FREE_ON_WINLDAP(host);
 
   /* no data to transfer */
-  Curl_xfer_setup(data, -1, -1, FALSE, -1);
+  Curl_xfer_setup_nop(data);
   connclose(conn, "LDAP connection always disable reuse");
 
   return result;

lib/openldap.c

@@ -921,7 +921,7 @@ static CURLcode oldap_do(struct Curl_easy *data, bool *done)
       else {
         lr->msgid = msgid;
         data->req.p.ldap = lr;
-        Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1);
+        Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE);
         *done = TRUE;
       }
     }

lib/pop3.c

@@ -936,7 +936,7 @@ static CURLcode pop3_state_command_resp(struct Curl_easy *data,
 
   if(pop3->transfer == PPTRANSFER_BODY) {
     /* POP3 download */
-    Curl_xfer_setup(data, FIRSTSOCKET, -1, FALSE, -1);
+    Curl_xfer_setup1(data, CURL_XFER_RECV, -1, FALSE);
 
     if(pp->overflow) {
       /* The recv buffer contains data that is actually body content so send

lib/request.c

@@ -54,6 +54,7 @@ CURLcode Curl_req_soft_reset(struct SingleRequest *req,
   req->upload_done = FALSE;
   req->download_done = FALSE;
   req->ignorebody = FALSE;
+  req->shutdown = FALSE;
   req->bytecount = 0;
   req->writebytecount = 0;
   req->header = TRUE; /* assume header */
@@ -156,6 +157,7 @@ void Curl_req_hard_reset(struct SingleRequest *req, struct Curl_easy *data)
   req->getheader = FALSE;
   req->no_body = data->set.opt_no_body;
   req->authneg = FALSE;
+  req->shutdown = FALSE;
 }
 
 void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data)
@@ -291,6 +293,14 @@ static CURLcode req_flush(struct Curl_easy *data)
 
   if(!data->req.upload_done && data->req.eos_read &&
      Curl_bufq_is_empty(&data->req.sendbuf)) {
+    if(data->req.shutdown) {
+      bool done;
+      result = Curl_xfer_send_shutdown(data, &done);
+      if(result)
+        return result;
+      if(!done)
+        return CURLE_AGAIN;
+    }
     return req_set_upload_done(data);
   }
   return CURLE_OK;

lib/request.h

@@ -147,6 +147,7 @@ struct SingleRequest {
                         but it is not the final request in the auth
                         negotiation. */
   BIT(sendbuf_init); /* sendbuf is initialized */
+  BIT(shutdown);     /* request end will shutdown connection */
 };
 
 /**

lib/rtsp.c

@@ -310,7 +310,7 @@ static CURLcode rtsp_do(struct Curl_easy *data, bool *done)
   }
 
   if(rtspreq == RTSPREQ_RECEIVE) {
-    Curl_xfer_setup(data, FIRSTSOCKET, -1, TRUE, -1);
+    Curl_xfer_setup1(data, CURL_XFER_RECV, -1, TRUE);
     goto out;
   }
 
@@ -578,7 +578,7 @@ static CURLcode rtsp_do(struct Curl_easy *data, bool *done)
   if(result)
     goto out;
 
-  Curl_xfer_setup(data, FIRSTSOCKET, -1, TRUE, FIRSTSOCKET);
+  Curl_xfer_setup1(data, CURL_XFER_SENDRECV, -1, TRUE);
 
   /* issue the request */
   result = Curl_req_send(data, &req_buffer);

lib/smtp.c

@@ -1164,7 +1164,7 @@ static CURLcode smtp_state_data_resp(struct Curl_easy *data, int smtpcode,
     Curl_pgrsSetUploadSize(data, data->state.infilesize);
 
     /* SMTP upload */
-    Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+    Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
     /* End of DO phase */
     smtp_state(data, SMTP_STOP);
@@ -1550,7 +1550,7 @@ static CURLcode smtp_dophase_done(struct Curl_easy *data, bool connected)
 
   if(smtp->transfer != PPTRANSFER_BODY)
     /* no data to transfer */
-    Curl_xfer_setup(data, -1, -1, FALSE, -1);
+    Curl_xfer_setup_nop(data);
 
   return CURLE_OK;
 }

lib/telnet.c

@@ -1645,7 +1645,7 @@ static CURLcode telnet_do(struct Curl_easy *data, bool *done)
   }
 #endif
   /* mark this as "no further transfer wanted" */
-  Curl_xfer_setup(data, -1, -1, FALSE, -1);
+  Curl_xfer_setup_nop(data);
 
   return result;
 }

lib/tftp.c

@@ -1242,7 +1242,7 @@ static CURLcode tftp_multi_statemach(struct Curl_easy *data, bool *done)
     *done = (state->state == TFTP_STATE_FIN) ? TRUE : FALSE;
     if(*done)
       /* Tell curl we're done */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
   }
   else {
     /* no timeouts to handle, check our socket */
@@ -1265,7 +1265,7 @@ static CURLcode tftp_multi_statemach(struct Curl_easy *data, bool *done)
       *done = (state->state == TFTP_STATE_FIN) ? TRUE : FALSE;
       if(*done)
         /* Tell curl we're done */
-        Curl_xfer_setup(data, -1, -1, FALSE, -1);
+        Curl_xfer_setup_nop(data);
     }
     /* if rc == 0, then select() timed out */
   }

lib/transfer.c

@@ -160,6 +160,30 @@ bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc)
   return TRUE;
 }
 
+static CURLcode xfer_recv_shutdown(struct Curl_easy *data, bool *done)
+{
+  int sockindex;
+
+  if(!data || !data->conn)
+    return CURLE_FAILED_INIT;
+  if(data->conn->sockfd == CURL_SOCKET_BAD)
+    return CURLE_FAILED_INIT;
+  sockindex = (data->conn->sockfd == data->conn->sock[SECONDARYSOCKET]);
+  return Curl_conn_shutdown(data, sockindex, done);
+}
+
+static bool xfer_recv_shutdown_started(struct Curl_easy *data)
+{
+  int sockindex;
+
+  if(!data || !data->conn)
+    return CURLE_FAILED_INIT;
+  if(data->conn->sockfd == CURL_SOCKET_BAD)
+    return CURLE_FAILED_INIT;
+  sockindex = (data->conn->sockfd == data->conn->sock[SECONDARYSOCKET]);
+  return Curl_shutdown_started(data, sockindex);
+}
+
 /**
  * Receive raw response data for the transfer.
  * @param data         the transfer
@@ -186,17 +210,35 @@ static ssize_t Curl_xfer_recv_resp(struct Curl_easy *data,
     else if(totalleft < (curl_off_t)blen)
       blen = (size_t)totalleft;
   }
-
+  else if(xfer_recv_shutdown_started(data)) {
-  if(!blen) {
+    /* we already reveived everything. Do not try more. */
-    /* want nothing - continue as if read nothing. */
+    blen = 0;
-    DEBUGF(infof(data, "readwrite_data: we're done"));
+  }
-    *err = CURLE_OK;
+
-    return 0;
+  if(!blen) {
+    /* want nothing more */
+    *err = CURLE_OK;
+    nread = 0;
+  }
+  else {
+    *err = Curl_xfer_recv(data, buf, blen, &nread);
   }
 
-  *err = Curl_xfer_recv(data, buf, blen, &nread);
   if(*err)
     return -1;
+  if(nread == 0) {
+    if(data->req.shutdown) {
+      bool done;
+      *err = xfer_recv_shutdown(data, &done);
+      if(*err)
+        return -1;
+      if(!done) {
+        *err = CURLE_AGAIN;
+        return -1;
+      }
+    }
+    DEBUGF(infof(data, "readwrite_data: we're done"));
+  }
   DEBUGASSERT(nread >= 0);
   return nread;
 }
@@ -1064,16 +1106,17 @@ CURLcode Curl_retry_request(struct Curl_easy *data, char **url)
 }
 
 /*
- * Curl_xfer_setup() is called to setup some basic properties for the
+ * xfer_setup() is called to setup basic properties for the transfer.
- * upcoming transfer.
  */
-void Curl_xfer_setup(
+static void xfer_setup(
   struct Curl_easy *data,   /* transfer */
   int sockindex,            /* socket index to read from or -1 */
   curl_off_t size,          /* -1 if unknown at this point */
   bool getheader,           /* TRUE if header parsing is wanted */
-  int writesockindex        /* socket index to write to, it may very well be
+  int writesockindex,       /* socket index to write to, it may very well be
                                the same we read from. -1 disables */
+  bool shutdown             /* shutdown connection at transfer end. Only
+                             * supported when sending OR receiving. */
   )
 {
   struct SingleRequest *k = &data->req;
@@ -1083,6 +1126,7 @@ void Curl_xfer_setup(
   DEBUGASSERT(conn != NULL);
   DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
   DEBUGASSERT((writesockindex <= 1) && (writesockindex >= -1));
+  DEBUGASSERT(!shutdown || (sockindex == -1) || (writesockindex == -1));
 
   if(conn->bits.multiplex || conn->httpversion >= 20 || want_send) {
     /* when multiplexing, the read/write sockets need to be the same! */
@@ -1100,9 +1144,10 @@ void Curl_xfer_setup(
     conn->writesockfd = writesockindex == -1 ?
       CURL_SOCKET_BAD:conn->sock[writesockindex];
   }
-  k->getheader = getheader;
 
+  k->getheader = getheader;
   k->size = size;
+  k->shutdown = shutdown;
 
   /* The code sequence below is placed in this function just because all
      necessary input is not always known in do_complete() as this function may
@@ -1125,6 +1170,33 @@ void Curl_xfer_setup(
 
 }
 
+void Curl_xfer_setup_nop(struct Curl_easy *data)
+{
+  xfer_setup(data, -1, -1, FALSE, -1, FALSE);
+}
+
+void Curl_xfer_setup1(struct Curl_easy *data,
+                      int send_recv,
+                      curl_off_t recv_size,
+                      bool getheader)
+{
+  int recv_index = (send_recv & CURL_XFER_RECV)? FIRSTSOCKET : -1;
+  int send_index = (send_recv & CURL_XFER_SEND)? FIRSTSOCKET : -1;
+  DEBUGASSERT((recv_index >= 0) || (recv_size == -1));
+  xfer_setup(data, recv_index, recv_size, getheader, send_index, FALSE);
+}
+
+void Curl_xfer_setup2(struct Curl_easy *data,
+                      int send_recv,
+                      curl_off_t recv_size,
+                      bool shutdown)
+{
+  int recv_index = (send_recv & CURL_XFER_RECV)? SECONDARYSOCKET : -1;
+  int send_index = (send_recv & CURL_XFER_SEND)? SECONDARYSOCKET : -1;
+  DEBUGASSERT((recv_index >= 0) || (recv_size == -1));
+  xfer_setup(data, recv_index, recv_size, FALSE, send_index, shutdown);
+}
+
 CURLcode Curl_xfer_write_resp(struct Curl_easy *data,
                               const char *buf, size_t blen,
                               bool is_eos)
@@ -1239,3 +1311,15 @@ CURLcode Curl_xfer_send_close(struct Curl_easy *data)
   Curl_conn_ev_data_done_send(data);
   return CURLE_OK;
 }
+
+CURLcode Curl_xfer_send_shutdown(struct Curl_easy *data, bool *done)
+{
+  int sockindex;
+
+  if(!data || !data->conn)
+    return CURLE_FAILED_INIT;
+  if(data->conn->writesockfd == CURL_SOCKET_BAD)
+    return CURLE_FAILED_INIT;
+  sockindex = (data->conn->writesockfd == data->conn->sock[SECONDARYSOCKET]);
+  return Curl_conn_shutdown(data, sockindex, done);
+}

lib/transfer.h

@@ -76,15 +76,37 @@ CURLcode Curl_xfer_write_resp(struct Curl_easy *data,
 CURLcode Curl_xfer_write_resp_hd(struct Curl_easy *data,
                                  const char *hd0, size_t hdlen, bool is_eos);
 
-/* This sets up a forthcoming transfer */
+#define CURL_XFER_NOP     (0)
-void Curl_xfer_setup(struct Curl_easy *data,
+#define CURL_XFER_RECV    (1<<(0))
-                     int sockindex,     /* socket index to read from or -1 */
+#define CURL_XFER_SEND    (1<<(1))
-                     curl_off_t size,   /* -1 if unknown at this point */
+#define CURL_XFER_SENDRECV (CURL_XFER_RECV|CURL_XFER_SEND)
-                     bool getheader,    /* TRUE if header parsing is wanted */
+
-                     int writesockindex /* socket index to write to. May be
+/**
-                                           the same we read from. -1
+ * The transfer is neither receiving nor sending now.
-                                           disables */
+ */
-  );
+void Curl_xfer_setup_nop(struct Curl_easy *data);
+
+/**
+ * The transfer will use socket 1 to send/recv. `recv_size` is
+ * the amount to receive or -1 if unknown. `getheader` indicates
+ * response header processing is expected.
+ */
+void Curl_xfer_setup1(struct Curl_easy *data,
+                      int send_recv,
+                      curl_off_t recv_size,
+                      bool getheader);
+
+/**
+ * The transfer will use socket 2 to send/recv. `recv_size` is
+ * the amount to receive or -1 if unknown. With `shutdown` being
+ * set, the transfer is only allowed to either send OR receive
+ * and the socket 2 connection will be shutdown at the end of
+ * the transfer. An unclean shutdown will fail the transfer.
+ */
+void Curl_xfer_setup2(struct Curl_easy *data,
+                      int send_recv,
+                      curl_off_t recv_size,
+                      bool shutdown);
 
 /**
  * Multi has set transfer to DONE. Last chance to trigger
@@ -111,5 +133,6 @@ CURLcode Curl_xfer_recv(struct Curl_easy *data,
                         ssize_t *pnrcvd);
 
 CURLcode Curl_xfer_send_close(struct Curl_easy *data);
+CURLcode Curl_xfer_send_shutdown(struct Curl_easy *data, bool *done);
 
 #endif /* HEADER_CURL_TRANSFER_H */

lib/url.c

@@ -3547,7 +3547,7 @@ static CURLcode create_conn(struct Curl_easy *data,
         (void)conn->handler->done(data, result, FALSE);
         goto out;
       }
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
     }
 
     /* since we skip do_init() */

lib/vssh/libssh.c

@@ -1350,7 +1350,7 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
         Curl_pgrsSetUploadSize(data, data->state.infilesize);
       }
       /* upload data */
-      Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+      Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->sockfd = conn->writesockfd;
@@ -1576,7 +1576,7 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
       sshc->sftp_dir = NULL;
 
       /* no data to transfer */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
       state(data, SSH_STOP);
       break;
 
@@ -1721,12 +1721,12 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
     /* Setup the actual download */
     if(data->req.size == 0) {
       /* no data to transfer */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
       infof(data, "File already completely downloaded");
       state(data, SSH_STOP);
       break;
     }
-    Curl_xfer_setup(data, FIRSTSOCKET, data->req.size, FALSE, -1);
+    Curl_xfer_setup1(data, CURL_XFER_RECV, data->req.size, FALSE);
 
     /* not set by Curl_xfer_setup to preserve keepon bits */
     conn->writesockfd = conn->sockfd;
@@ -1850,7 +1850,7 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
       }
 
       /* upload data */
-      Curl_xfer_setup(data, -1, data->req.size, FALSE, FIRSTSOCKET);
+      Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->sockfd = conn->writesockfd;
@@ -1894,7 +1894,7 @@ static CURLcode myssh_statemach_act(struct Curl_easy *data, bool *block)
         /* download data */
         bytecount = ssh_scp_request_get_size(sshc->scp_session);
         data->req.maxdownload = (curl_off_t) bytecount;
-        Curl_xfer_setup(data, FIRSTSOCKET, bytecount, FALSE, -1);
+        Curl_xfer_setup1(data, CURL_XFER_RECV, bytecount, FALSE);
 
         /* not set by Curl_xfer_setup to preserve keepon bits */
         conn->writesockfd = conn->sockfd;

lib/vssh/libssh2.c

@@ -2199,7 +2199,7 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
         Curl_pgrsSetUploadSize(data, data->state.infilesize);
       }
       /* upload data */
-      Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+      Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->sockfd = conn->writesockfd;
@@ -2453,7 +2453,7 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
       Curl_safefree(sshp->readdir_longentry);
 
       /* no data to transfer */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
       state(data, SSH_STOP);
       break;
 
@@ -2595,12 +2595,12 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
     /* Setup the actual download */
     if(data->req.size == 0) {
       /* no data to transfer */
-      Curl_xfer_setup(data, -1, -1, FALSE, -1);
+      Curl_xfer_setup_nop(data);
       infof(data, "File already completely downloaded");
       state(data, SSH_STOP);
       break;
     }
-    Curl_xfer_setup(data, FIRSTSOCKET, data->req.size, FALSE, -1);
+    Curl_xfer_setup1(data, CURL_XFER_RECV, data->req.size, FALSE);
 
     /* not set by Curl_xfer_setup to preserve keepon bits */
     conn->writesockfd = conn->sockfd;
@@ -2746,7 +2746,7 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
       /* upload data */
       data->req.size = data->state.infilesize;
       Curl_pgrsSetUploadSize(data, data->state.infilesize);
-      Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+      Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->sockfd = conn->writesockfd;
@@ -2817,7 +2817,7 @@ static CURLcode ssh_statemach_act(struct Curl_easy *data, bool *block)
       /* download data */
       bytecount = (curl_off_t)sb.st_size;
       data->req.maxdownload = (curl_off_t)sb.st_size;
-      Curl_xfer_setup(data, FIRSTSOCKET, bytecount, FALSE, -1);
+      Curl_xfer_setup1(data, CURL_XFER_RECV, bytecount, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->writesockfd = conn->sockfd;

lib/vssh/wolfssh.c

@@ -680,7 +680,7 @@ static CURLcode wssh_statemach_act(struct Curl_easy *data, bool *block)
         Curl_pgrsSetUploadSize(data, data->state.infilesize);
       }
       /* upload data */
-      Curl_xfer_setup(data, -1, -1, FALSE, FIRSTSOCKET);
+      Curl_xfer_setup1(data, CURL_XFER_SEND, -1, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->sockfd = conn->writesockfd;
@@ -780,12 +780,12 @@ static CURLcode wssh_statemach_act(struct Curl_easy *data, bool *block)
       /* Setup the actual download */
       if(data->req.size == 0) {
         /* no data to transfer */
-        Curl_xfer_setup(data, -1, -1, FALSE, -1);
+        Curl_xfer_setup_nop(data);
         infof(data, "File already completely downloaded");
         state(data, SSH_STOP);
         break;
       }
-      Curl_xfer_setup(data, FIRSTSOCKET, data->req.size, FALSE, -1);
+      Curl_xfer_setup1(data, CURL_XFER_RECV, data->req.size, FALSE);
 
       /* not set by Curl_xfer_setup to preserve keepon bits */
       conn->writesockfd = conn->sockfd;

lib/vtls/gtls.c

@@ -1829,6 +1829,13 @@ static CURLcode gtls_shutdown(struct Curl_cfilter *cf,
     backend->gtls.sent_shutdown = TRUE;
     if(send_shutdown) {
       int ret = gnutls_bye(backend->gtls.session, GNUTLS_SHUT_RDWR);
+      if((ret == GNUTLS_E_AGAIN) || (ret == GNUTLS_E_INTERRUPTED)) {
+        CURL_TRC_CF(data, cf, "SSL shutdown, gnutls_bye EAGAIN");
+        connssl->io_need = gnutls_record_get_direction(backend->gtls.session)?
+          CURL_SSL_IO_NEED_SEND : CURL_SSL_IO_NEED_RECV;
+        result = CURLE_OK;
+        goto out;
+      }
       if(ret != GNUTLS_E_SUCCESS) {
         CURL_TRC_CF(data, cf, "SSL shutdown, gnutls_bye error: '%s'(%d)",
                     gnutls_strerror((int)ret), (int)ret);

lib/vtls/schannel.c

@@ -2499,7 +2499,12 @@ static CURLcode schannel_shutdown(struct Curl_cfilter *cf,
           connssl->peer.hostname, connssl->peer.port);
   }
 
-  if(backend->cred && backend->ctxt) {
+  if(!backend->ctxt || connssl->shutdown) {
+    *done = TRUE;
+    goto out;
+  }
+
+  if(backend->cred && backend->ctxt && !backend->sent_shutdown) {
     SecBufferDesc BuffDesc;
     SecBuffer Buffer;
     SECURITY_STATUS sspi_status;
@@ -2545,11 +2550,58 @@ static CURLcode schannel_shutdown(struct Curl_cfilter *cf,
                                           outbuf.pvBuffer, outbuf.cbBuffer,
                                           &result);
       s_pSecFn->FreeContextBuffer(outbuf.pvBuffer);
-      if((result != CURLE_OK) || (outbuf.cbBuffer != (size_t) written)) {
+      if(!result) {
-        infof(data, "schannel: failed to send close msg: %s"
+        if(written < (ssize_t)outbuf.cbBuffer) {
-              " (bytes written: %zd)", curl_easy_strerror(result), written);
+          /* TODO: handle partial sends */
-        result = CURLE_SEND_ERROR;
+          infof(data, "schannel: failed to send close msg: %s"
+                " (bytes written: %zd)", curl_easy_strerror(result), written);
+          result = CURLE_SEND_ERROR;
+          goto out;
+        }
+        backend->sent_shutdown = TRUE;
+        *done = TRUE;
       }
+      else if(result == CURLE_AGAIN) {
+        connssl->io_need = CURL_SSL_IO_NEED_SEND;
+        result = CURLE_OK;
+        goto out;
+      }
+      else {
+        if(!backend->recv_connection_closed) {
+          infof(data, "schannel: error sending close msg: %d", result);
+          result = CURLE_SEND_ERROR;
+          goto out;
+        }
+        /* Looks like server already closed the connection.
+         * An error to send our close notify is not a failure. */
+        *done = TRUE;
+        result = CURLE_OK;
+      }
+    }
+  }
+
+  /* If the connection seems open and we have not seen the close notify
+   * from the server yet, try to receive it. */
+  if(backend->cred && backend->ctxt &&
+     !backend->recv_sspi_close_notify && !backend->recv_connection_closed) {
+    char buffer[1024];
+    ssize_t nread;
+
+    nread = schannel_recv(cf, data, buffer, sizeof(buffer), &result);
+    if(nread > 0) {
+      /* still data coming in? */
+    }
+    else if(nread == 0) {
+      /* We got the close notify alert and are done. */
+      backend->recv_connection_closed = TRUE;
+      *done = TRUE;
+    }
+    else if(nread < 0 && result == CURLE_AGAIN) {
+      connssl->io_need = CURL_SSL_IO_NEED_RECV;
+    }
+    else {
+      CURL_TRC_CF(data, cf, "SSL shutdown, error %d", result);
+      result = CURLE_RECV_ERROR;
     }
   }
 

lib/vtls/schannel_int.h

@@ -158,6 +158,7 @@ struct schannel_ssl_backend_data {
 #ifdef HAS_MANUAL_VERIFY_API
   bool use_manual_cred_validation; /* true if manual cred validation is used */
 #endif
+  BIT(sent_shutdown);
 };
 
 /* key to use at `multi->proto_hash` */