curl: improved IPFS and IPNS URL support

Previously just ipfs://<cid> and ipns://<cid> was supported, which is too strict for some usecases. This patch allows paths and query arguments to be used too. Making this work according to normal http semantics: ipfs://<cid>/foo/bar?key=val ipns://<cid>/foo/bar?key=val The gateway url support is changed. It now only supports gateways in the form of: http://<gateway>/foo/bar http://<gateway> Query arguments here are explicitly not allowed and trigger an intended malformed url error. There also was a crash when IPFS_PATH was set with a non trailing forward slash. This has been fixed. Lastly, a load of test cases have been added to verify the above. Reported-by: Steven Allen Fixes #12148 Closes #12152
2023-10-18 02:28:06 +02:00 · 2023-10-18 02:28:06 +02:00 · 859e88f653
commit 859e88f653
parent d112c24234
21 changed files with 765 additions and 53 deletions
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@ -697,6 +697,32 @@ noretry:
  return result;
 }

+/* helper function to ensure input ends in val_to_ensure */
+static CURLcode ensure_trailing(char **input, const char val_to_ensure)
+{
+  if(*input && **input) {
+    size_t len = strlen(*input);
+    if(((*input)[len - 1] != val_to_ensure)) {
+      struct curlx_dynbuf dyn;
+      curlx_dyn_init(&dyn, len + 2);
+
+      if(curlx_dyn_addn(&dyn, *input, len)) {
+        Curl_safefree(*input);
+        return CURLE_OUT_OF_MEMORY;
+      }
+
+      Curl_safefree(*input);
+
+      if(curlx_dyn_addn(&dyn, &val_to_ensure, 1))
+        return CURLE_OUT_OF_MEMORY;
+
+      *input = curlx_dyn_ptr(&dyn);
+    }
+  }
+
+  return CURLE_OK;
+}
+
 static char *ipfs_gateway(void)
 {
  char *gateway = NULL;
@ -704,31 +730,27 @@ static char *ipfs_gateway(void)
  char *gateway_composed_file_path = NULL;
  FILE *gateway_file = NULL;

-  gateway = getenv("IPFS_GATEWAY");
+  gateway = curlx_getenv("IPFS_GATEWAY");

  /* Gateway is found from environment variable. */
-  if(gateway && *gateway) {
-    char *composed_gateway = NULL;
-    bool add_slash = (gateway[strlen(gateway) - 1] != '/');
-    composed_gateway = aprintf("%s%s", gateway, (add_slash) ? "/" : "");
-    if(composed_gateway) {
-      gateway = aprintf("%s", composed_gateway);
-      Curl_safefree(composed_gateway);
+  if(gateway) {
+    if(ensure_trailing(&gateway, '/')) {
+      Curl_safefree(gateway);
+      return NULL;
    }
    return gateway;
  }
-  else
-    /* a blank string does not count */
-    gateway = NULL;

  /* Try to find the gateway in the IPFS data folder. */
-  ipfs_path = getenv("IPFS_PATH");
+  ipfs_path = curlx_getenv("IPFS_PATH");

  if(!ipfs_path) {
-    char *home = getenv("HOME");
+    char *home = curlx_getenv("HOME");
    if(home && *home)
      ipfs_path = aprintf("%s/.ipfs/", home);
    /* fallback to "~/.ipfs", as that's the default location. */
+
+    Curl_safefree(home);
  }

  if(!ipfs_path) {
@ -736,6 +758,13 @@ static char *ipfs_gateway(void)
    Curl_safefree(ipfs_path);
    return NULL;
  }
+  else {
+    if(ensure_trailing(&ipfs_path, '/')) {
+      Curl_safefree(gateway);
+      Curl_safefree(ipfs_path);
+      return NULL;
+    }
+  }

  gateway_composed_file_path = aprintf("%sgateway", ipfs_path);

@ -749,24 +778,32 @@ static char *ipfs_gateway(void)
  Curl_safefree(gateway_composed_file_path);

  if(gateway_file) {
-    char *buf = NULL;
+    int c;
+    struct curlx_dynbuf dyn;
+    curlx_dyn_init(&dyn, INT_MAX);

-    if((PARAM_OK == file2string(&buf, gateway_file)) && buf && *buf) {
-      bool add_slash = (buf[strlen(buf) - 1] != '/');
-      gateway = aprintf("%s%s", buf, (add_slash) ? "/" : "");
+    /* get the first line of the gateway file, ignore the rest */
+    while((c = getc(gateway_file)) != EOF && c != '\n' && c != '\r') {
+      if(curlx_dyn_addn(&dyn, &c, 1))
+        break;
    }
-    Curl_safefree(buf);

    if(gateway_file)
      fclose(gateway_file);

+    if(curlx_dyn_len(&dyn) > 0)
+      gateway = curlx_dyn_ptr(&dyn);
+
+    if(gateway)
+      ensure_trailing(&gateway, '/');
+
    if(!gateway) {
-      Curl_safefree(gateway);
      Curl_safefree(ipfs_path);
      return NULL;
    }

    Curl_safefree(ipfs_path);
+
    return gateway;
  }

@ -783,20 +820,26 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
                                 struct OperationConfig *config)
 {
  CURLcode result = CURLE_URL_MALFORMAT;
-  CURLUcode urlGetResult;
+  CURLUcode getResult;
  char *gateway = NULL;
+  char *gatewayhost = NULL;
+  char *gatewaypath = NULL;
+  char *gatewayquery = NULL;
+  char *inputpath = NULL;
+  char *gatewayscheme = NULL;
+  char *gatewayport = NULL;
  char *cid = NULL;
  char *pathbuffer = NULL;
-  CURLU *ipfsurl = curl_url();
+  CURLU *gatewaysurl = curl_url();

-  if(!ipfsurl) {
+  if(!gatewaysurl) {
    result = CURLE_FAILED_INIT;
    goto clean;
  }

-  urlGetResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE);
+  getResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE);

-  if(urlGetResult) {
+  if(getResult) {
    goto clean;
  }

@ -808,9 +851,14 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
   * if we do have something but if it's an invalid url.
   */
  if(config->ipfs_gateway) {
-    if(curl_url_set(ipfsurl, CURLUPART_URL, config->ipfs_gateway,
-                    CURLU_GUESS_SCHEME)
-                    == CURLUE_OK) {
+    /* ensure the gateway ends in a trailing / */
+    if(ensure_trailing(&config->ipfs_gateway, '/') != CURLE_OK) {
+      result = CURLE_OUT_OF_MEMORY;
+      goto clean;
+    }
+
+    if(!curl_url_set(gatewaysurl, CURLUPART_URL, config->ipfs_gateway,
+                    CURLU_GUESS_SCHEME)) {
      gateway = strdup(config->ipfs_gateway);
      if(!gateway) {
        result = CURLE_URL_MALFORMAT;
@ -824,33 +872,83 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
    }
  }
  else {
+    /* this is ensured to end in a trailing / within ipfs_gateway() */
    gateway = ipfs_gateway();
    if(!gateway) {
      result = CURLE_FILE_COULDNT_READ_FILE;
      goto clean;
    }

-    if(curl_url_set(ipfsurl, CURLUPART_URL, gateway, CURLU_GUESS_SCHEME
-                    | CURLU_NON_SUPPORT_SCHEME) != CURLUE_OK) {
+    if(curl_url_set(gatewaysurl, CURLUPART_URL, gateway, 0)) {
+      result = CURLE_URL_MALFORMAT;
      goto clean;
    }
  }

-  pathbuffer = aprintf("%s/%s", protocol, cid);
+  /* check for unsupported gateway parts */
+  if(curl_url_get(gatewaysurl, CURLUPART_QUERY, &gatewayquery, 0)
+                  != CURLUE_NO_QUERY) {
+    result = CURLE_URL_MALFORMAT;
+    goto clean;
+  }
+
+  /* get gateway parts */
+  if(curl_url_get(gatewaysurl, CURLUPART_HOST,
+                  &gatewayhost, CURLU_URLDECODE)) {
+    goto clean;
+  }
+
+  if(curl_url_get(gatewaysurl, CURLUPART_SCHEME,
+                  &gatewayscheme, CURLU_URLDECODE)) {
+    goto clean;
+  }
+
+  curl_url_get(gatewaysurl, CURLUPART_PORT, &gatewayport, CURLU_URLDECODE);
+  curl_url_get(gatewaysurl, CURLUPART_PATH, &gatewaypath, CURLU_URLDECODE);
+
+  /* get the path from user input */
+  if(curl_url_get(uh, CURLUPART_PATH, &inputpath, CURLU_URLDECODE)) {
+    inputpath = strdup("");
+    if(!inputpath)
+      goto clean;
+  }
+
+  /* set gateway parts in input url */
+  if(curl_url_set(uh, CURLUPART_SCHEME, gatewayscheme, CURLU_URLENCODE)) {
+    goto clean;
+  }
+
+  if(curl_url_set(uh, CURLUPART_HOST, gatewayhost, CURLU_URLENCODE)) {
+    goto clean;
+  }
+
+  if(curl_url_set(uh, CURLUPART_PORT, gatewayport, CURLU_URLENCODE)) {
+    goto clean;
+  }
+
+  /* if the input path is just a slash, clear it */
+  if(inputpath && *inputpath && strlen(inputpath) == 1) {
+    if(*inputpath == '/') {
+      *inputpath = '\0';
+    }
+  }
+
+  /* ensure the gateway path ends with a trailing slash */
+  ensure_trailing(&gatewaypath, '/');
+
+  pathbuffer = aprintf("%s%s/%s%s", gatewaypath, protocol, cid, inputpath);
  if(!pathbuffer) {
    goto clean;
  }

-  if(curl_url_set(ipfsurl, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)
-                  != CURLUE_OK) {
+  if(curl_url_set(uh, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)) {
    goto clean;
  }

  /* Free whatever it has now, rewriting is next */
  Curl_safefree(*url);

-  if(curl_url_get(ipfsurl, CURLUPART_URL, url, CURLU_URLENCODE)
-                  != CURLUE_OK) {
+  if(curl_url_get(uh, CURLUPART_URL, url, CURLU_URLENCODE)) {
    goto clean;
  }

@ -858,9 +956,15 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,

 clean:
  free(gateway);
+  curl_free(gatewayhost);
+  curl_free(gatewaypath);
+  curl_free(gatewayquery);
+  curl_free(inputpath);
+  curl_free(gatewayscheme);
+  curl_free(gatewayport);
  curl_free(cid);
  curl_free(pathbuffer);
-  curl_url_cleanup(ipfsurl);
+  curl_url_cleanup(gatewaysurl);

  switch(result) {
  case CURLE_URL_MALFORMAT:
--- a/tests/data/DISABLED
+++ b/tests/data/DISABLED
@ -70,9 +70,6 @@
 266
 579
 587
-722
-724
-727
 # 1021 re-added here due to flakiness
 1021
 1117
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@ -101,7 +101,8 @@ test681 test682 test683 test684 test685 test686 test687 test688 \
 test700 test701 test702 test703 test704 test705 test706 test707 test708 \
 test709 test710 test711 test712 test713 test714 test715 test716 test717 \
 test718 test719 test720 test721 test722 test723 test724 test725 test726 \
-test727 test728 test729 \
+test727 test728 test729 test730 test731 test732 test733 test734 test735 \
+test736 test737 test738 test739 test740 test741 \
 \
 test799 test800 test801 test802 test803 test804 test805 test806 test807 \
 test808 test809 test810 test811 test812 test813 test814 test815 test816 \
--- a/tests/data/test722
+++ b/tests/data/test722
@ -8,7 +8,7 @@ IPFS
 #
 # Server-side
 <reply>
-<data>
+<data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
@ -34,7 +34,7 @@ http
 IPFS
 </name>
 <command>
--ipfs-gateway http://%HOSTIP:%HTTPPORT/%TESTNUMBER ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+--ipfs-gateway http://%HOSTIP:%HTTPPORT ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>

@ -42,7 +42,7 @@ IPFS
 # Verify data after the test has been "shot"
 <verify>
 <protocol crlf="yes">
-GET /ipfs/QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx HTTP/1.1
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*
--- a/tests/data/test723
+++ b/tests/data/test723
@ -20,7 +20,7 @@ http
 IPFS with malformed gateway URL (bad function argument error)
 </name>
 <command>
--ipfs-gateway http://nonexisting,local:8080/%TESTNUMBER ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+--ipfs-gateway http://nonexisting,local:8080 ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>

--- a/tests/data/test724
+++ b/tests/data/test724
@ -8,7 +8,7 @@ IPFS
 #
 # Server-side
 <reply>
-<data>
+<data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
@ -37,10 +37,10 @@ HOME=%PWD/%LOGDIR
 IPFS with gateway URL from gateway file
 </name>
 <command>
-ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 <file name="%LOGDIR/.ipfs/gateway" >
-http://%HOSTIP:%HTTPPORT/%TESTNUMBER
+http://%HOSTIP:%HTTPPORT
 </file>
 </client>

@ -48,7 +48,7 @@ http://%HOSTIP:%HTTPPORT/%TESTNUMBER
 # Verify data after the test has been "shot"
 <verify>
 <protocol crlf="yes">
-GET /ipfs/QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx HTTP/1.1
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*
--- a/tests/data/test725
+++ b/tests/data/test725
@ -23,10 +23,10 @@ HOME=%PWD/%LOGDIR
 IPFS with malformed gateway URL from gateway file
 </name>
 <command>
-ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 <file name="%LOGDIR/.ipfs/gateway" >
-http://nonexisting,local:8080/%TESTNUMBER
+http://nonexisting,local:8080
 </file>
 </client>

--- a/tests/data/test726
+++ b/tests/data/test726
@ -26,7 +26,7 @@ HOME=%PWD
 IPFS with no gateway URL (no environment or home file either)
 </name>
 <command>
-ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>

--- a/tests/data/test727
+++ b/tests/data/test727
@ -8,7 +8,7 @@ IPNS
 #
 # Server-side
 <reply>
-<data>
+<data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
@ -34,7 +34,7 @@ http
 IPNS
 </name>
 <command>
--ipfs-gateway http://%HOSTIP:%HTTPPORT/%TESTNUMBER ipns://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+--ipfs-gateway http://%HOSTIP:%HTTPPORT ipns://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>

@ -42,7 +42,7 @@ IPNS
 # Verify data after the test has been "shot"
 <verify>
 <protocol crlf="yes">
-GET /ipns/QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx HTTP/1.1
+GET /ipns/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*
--- a/tests/data/test730
+++ b/tests/data/test730
@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS arg gateway with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT/foo/bar ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /foo/bar/ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test731
+++ b/tests/data/test731
@ -0,0 +1,58 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+</setenv>
+<name>
+IPFS with gateway URL and path from gateway file
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /%TESTNUMBER/ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test732
+++ b/tests/data/test732
@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT "ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test733
+++ b/tests/data/test733
@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS with path and query args
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT "ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test734
+++ b/tests/data/test734
@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS with path, query args and gateway with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT/some/path "ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /some/path/ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test735
+++ b/tests/data/test735
@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPNS with path, query args and gateway with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT/some/path "ipns://fancy.tld/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /some/path/ipns/fancy.tld/a/b?foo=bar&aaa=bbb HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test736
+++ b/tests/data/test736
@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+IPFS_DATA=$HOME/.ipfs
+</setenv>
+<name>
+IPFS with IPFS_DATA set, no traling slash
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test737
+++ b/tests/data/test737
@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+IPFS_DATA=$HOME/.ipfs/
+</setenv>
+<name>
+IPFS with IPFS_DATA set, with traling slash
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test738
+++ b/tests/data/test738
@ -0,0 +1,38 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+IPFS_DATA=%HOME/.ipfs/
+</setenv>
+<name>
+IPFS with IPFS_DATA, no gateway file
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+</client>
+
+#
+# Verify error code with no gateway file (detection fails)
+<verify>
+<errorcode>
+37
+</errorcode>
+</verify>
+</testcase>
--- a/tests/data/test739
+++ b/tests/data/test739
@ -0,0 +1,34 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPNS path and query args for gateway and IPFS url (malformed gateway url)
+</name>
+<command>
+--ipfs-gateway "http://%HOSTIP:%HTTPPORT/some/path?biz=baz" "ipns://fancy.tld/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+3
+</errorcode>
+</verify>
+</testcase>
--- a/tests/data/test740
+++ b/tests/data/test740
@ -0,0 +1,60 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+</setenv>
+<name>
+IPFS with gateway URL from multiline gateway file
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT
+foo
+bar
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>
--- a/tests/data/test741
+++ b/tests/data/test741
@ -0,0 +1,42 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+</setenv>
+<name>
+IPFS with malformed gateway URL from multiline gateway file, first line no url
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+foo
+bar
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+# malformed gateway URL, first line in file must be a gateway URL
+<errorcode>
+3
+</errorcode>
+</verify>
+</testcase>