diff --git a/lib/transfer.c b/lib/transfer.c
index cd3bd37682..669beebd3a 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -242,13 +242,14 @@ static CURLcode readwrite_data(struct Curl_easy *data,
     buf = xfer_buf;
     bytestoread = xfer_blen;
 
-    /* Observe any imposed speed limit */
     if(bytestoread && data->set.max_recv_speed) {
-      curl_off_t net_limit = data->set.max_recv_speed - total_received;
-      if(net_limit <= 0)
+      /* In case of speed limit on receiving: if this loop already got
+       * data, break out. If not, limit the amount of bytes to receive.
+       * The overall, timed, speed limiting is done in multi.c */
+      if(total_received)
         break;
-      if((size_t)net_limit < bytestoread)
-        bytestoread = (size_t)net_limit;
+      if((size_t)data->set.max_recv_speed < bytestoread)
+        bytestoread = (size_t)data->set.max_recv_speed;
     }
 
     nread = Curl_xfer_recv_resp(data, buf, bytestoread,