curl: improved IPFS and IPNS URL support

Previously just ipfs://<cid> and ipns://<cid> was supported, which is
too strict for some usecases.

This patch allows paths and query arguments to be used too.
Making this work according to normal http semantics:

 ipfs://<cid>/foo/bar?key=val
 ipns://<cid>/foo/bar?key=val

The gateway url support is changed.
It now only supports gateways in the form of:

 http://<gateway>/foo/bar
 http://<gateway>

Query arguments here are explicitly not allowed and trigger an intended
malformed url error.

There also was a crash when IPFS_PATH was set with a non trailing
forward slash. This has been fixed.

Lastly, a load of test cases have been added to verify the above.

Reported-by: Steven Allen
Fixes #12148
Closes #12152

src/tool_operate.c

@@ -697,6 +697,32 @@ noretry:
   return result;
 }
 
+/* helper function to ensure input ends in val_to_ensure */
+static CURLcode ensure_trailing(char **input, const char val_to_ensure)
+{
+  if(*input && **input) {
+    size_t len = strlen(*input);
+    if(((*input)[len - 1] != val_to_ensure)) {
+      struct curlx_dynbuf dyn;
+      curlx_dyn_init(&dyn, len + 2);
+
+      if(curlx_dyn_addn(&dyn, *input, len)) {
+        Curl_safefree(*input);
+        return CURLE_OUT_OF_MEMORY;
+      }
+
+      Curl_safefree(*input);
+
+      if(curlx_dyn_addn(&dyn, &val_to_ensure, 1))
+        return CURLE_OUT_OF_MEMORY;
+
+      *input = curlx_dyn_ptr(&dyn);
+    }
+  }
+
+  return CURLE_OK;
+}
+
 static char *ipfs_gateway(void)
 {
   char *gateway = NULL;
@@ -704,31 +730,27 @@ static char *ipfs_gateway(void)
   char *gateway_composed_file_path = NULL;
   FILE *gateway_file = NULL;
 
-  gateway = getenv("IPFS_GATEWAY");
+  gateway = curlx_getenv("IPFS_GATEWAY");
 
   /* Gateway is found from environment variable. */
-  if(gateway && *gateway) {
+  if(gateway) {
-    char *composed_gateway = NULL;
+    if(ensure_trailing(&gateway, '/')) {
-    bool add_slash = (gateway[strlen(gateway) - 1] != '/');
+      Curl_safefree(gateway);
-    composed_gateway = aprintf("%s%s", gateway, (add_slash) ? "/" : "");
+      return NULL;
-    if(composed_gateway) {
-      gateway = aprintf("%s", composed_gateway);
-      Curl_safefree(composed_gateway);
     }
     return gateway;
   }
-  else
-    /* a blank string does not count */
-    gateway = NULL;
 
   /* Try to find the gateway in the IPFS data folder. */
-  ipfs_path = getenv("IPFS_PATH");
+  ipfs_path = curlx_getenv("IPFS_PATH");
 
   if(!ipfs_path) {
-    char *home = getenv("HOME");
+    char *home = curlx_getenv("HOME");
     if(home && *home)
       ipfs_path = aprintf("%s/.ipfs/", home);
     /* fallback to "~/.ipfs", as that's the default location. */
+
+    Curl_safefree(home);
   }
 
   if(!ipfs_path) {
@@ -736,6 +758,13 @@ static char *ipfs_gateway(void)
     Curl_safefree(ipfs_path);
     return NULL;
   }
+  else {
+    if(ensure_trailing(&ipfs_path, '/')) {
+      Curl_safefree(gateway);
+      Curl_safefree(ipfs_path);
+      return NULL;
+    }
+  }
 
   gateway_composed_file_path = aprintf("%sgateway", ipfs_path);
 
@@ -749,24 +778,32 @@ static char *ipfs_gateway(void)
   Curl_safefree(gateway_composed_file_path);
 
   if(gateway_file) {
-    char *buf = NULL;
+    int c;
+    struct curlx_dynbuf dyn;
+    curlx_dyn_init(&dyn, INT_MAX);
 
-    if((PARAM_OK == file2string(&buf, gateway_file)) && buf && *buf) {
+    /* get the first line of the gateway file, ignore the rest */
-      bool add_slash = (buf[strlen(buf) - 1] != '/');
+    while((c = getc(gateway_file)) != EOF && c != '\n' && c != '\r') {
-      gateway = aprintf("%s%s", buf, (add_slash) ? "/" : "");
+      if(curlx_dyn_addn(&dyn, &c, 1))
+        break;
     }
-    Curl_safefree(buf);
 
     if(gateway_file)
       fclose(gateway_file);
 
+    if(curlx_dyn_len(&dyn) > 0)
+      gateway = curlx_dyn_ptr(&dyn);
+
+    if(gateway)
+      ensure_trailing(&gateway, '/');
+
     if(!gateway) {
-      Curl_safefree(gateway);
       Curl_safefree(ipfs_path);
       return NULL;
     }
 
     Curl_safefree(ipfs_path);
+
     return gateway;
   }
 
@@ -783,20 +820,26 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
                                  struct OperationConfig *config)
 {
   CURLcode result = CURLE_URL_MALFORMAT;
-  CURLUcode urlGetResult;
+  CURLUcode getResult;
   char *gateway = NULL;
+  char *gatewayhost = NULL;
+  char *gatewaypath = NULL;
+  char *gatewayquery = NULL;
+  char *inputpath = NULL;
+  char *gatewayscheme = NULL;
+  char *gatewayport = NULL;
   char *cid = NULL;
   char *pathbuffer = NULL;
-  CURLU *ipfsurl = curl_url();
+  CURLU *gatewaysurl = curl_url();
 
-  if(!ipfsurl) {
+  if(!gatewaysurl) {
     result = CURLE_FAILED_INIT;
     goto clean;
   }
 
-  urlGetResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE);
+  getResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE);
 
-  if(urlGetResult) {
+  if(getResult) {
     goto clean;
   }
 
@@ -808,9 +851,14 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
    * if we do have something but if it's an invalid url.
    */
   if(config->ipfs_gateway) {
-    if(curl_url_set(ipfsurl, CURLUPART_URL, config->ipfs_gateway,
+    /* ensure the gateway ends in a trailing / */
-                    CURLU_GUESS_SCHEME)
+    if(ensure_trailing(&config->ipfs_gateway, '/') != CURLE_OK) {
-                    == CURLUE_OK) {
+      result = CURLE_OUT_OF_MEMORY;
+      goto clean;
+    }
+
+    if(!curl_url_set(gatewaysurl, CURLUPART_URL, config->ipfs_gateway,
+                    CURLU_GUESS_SCHEME)) {
       gateway = strdup(config->ipfs_gateway);
       if(!gateway) {
         result = CURLE_URL_MALFORMAT;
@@ -824,33 +872,83 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
     }
   }
   else {
+    /* this is ensured to end in a trailing / within ipfs_gateway() */
     gateway = ipfs_gateway();
     if(!gateway) {
       result = CURLE_FILE_COULDNT_READ_FILE;
       goto clean;
     }
 
-    if(curl_url_set(ipfsurl, CURLUPART_URL, gateway, CURLU_GUESS_SCHEME
+    if(curl_url_set(gatewaysurl, CURLUPART_URL, gateway, 0)) {
-                    | CURLU_NON_SUPPORT_SCHEME) != CURLUE_OK) {
+      result = CURLE_URL_MALFORMAT;
       goto clean;
     }
   }
 
-  pathbuffer = aprintf("%s/%s", protocol, cid);
+  /* check for unsupported gateway parts */
+  if(curl_url_get(gatewaysurl, CURLUPART_QUERY, &gatewayquery, 0)
+                  != CURLUE_NO_QUERY) {
+    result = CURLE_URL_MALFORMAT;
+    goto clean;
+  }
+
+  /* get gateway parts */
+  if(curl_url_get(gatewaysurl, CURLUPART_HOST,
+                  &gatewayhost, CURLU_URLDECODE)) {
+    goto clean;
+  }
+
+  if(curl_url_get(gatewaysurl, CURLUPART_SCHEME,
+                  &gatewayscheme, CURLU_URLDECODE)) {
+    goto clean;
+  }
+
+  curl_url_get(gatewaysurl, CURLUPART_PORT, &gatewayport, CURLU_URLDECODE);
+  curl_url_get(gatewaysurl, CURLUPART_PATH, &gatewaypath, CURLU_URLDECODE);
+
+  /* get the path from user input */
+  if(curl_url_get(uh, CURLUPART_PATH, &inputpath, CURLU_URLDECODE)) {
+    inputpath = strdup("");
+    if(!inputpath)
+      goto clean;
+  }
+
+  /* set gateway parts in input url */
+  if(curl_url_set(uh, CURLUPART_SCHEME, gatewayscheme, CURLU_URLENCODE)) {
+    goto clean;
+  }
+
+  if(curl_url_set(uh, CURLUPART_HOST, gatewayhost, CURLU_URLENCODE)) {
+    goto clean;
+  }
+
+  if(curl_url_set(uh, CURLUPART_PORT, gatewayport, CURLU_URLENCODE)) {
+    goto clean;
+  }
+
+  /* if the input path is just a slash, clear it */
+  if(inputpath && *inputpath && strlen(inputpath) == 1) {
+    if(*inputpath == '/') {
+      *inputpath = '\0';
+    }
+  }
+
+  /* ensure the gateway path ends with a trailing slash */
+  ensure_trailing(&gatewaypath, '/');
+
+  pathbuffer = aprintf("%s%s/%s%s", gatewaypath, protocol, cid, inputpath);
   if(!pathbuffer) {
     goto clean;
   }
 
-  if(curl_url_set(ipfsurl, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)
+  if(curl_url_set(uh, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)) {
-                  != CURLUE_OK) {
     goto clean;
   }
 
   /* Free whatever it has now, rewriting is next */
   Curl_safefree(*url);
 
-  if(curl_url_get(ipfsurl, CURLUPART_URL, url, CURLU_URLENCODE)
+  if(curl_url_get(uh, CURLUPART_URL, url, CURLU_URLENCODE)) {
-                  != CURLUE_OK) {
     goto clean;
   }
 
@@ -858,9 +956,15 @@ static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
 
 clean:
   free(gateway);
+  curl_free(gatewayhost);
+  curl_free(gatewaypath);
+  curl_free(gatewayquery);
+  curl_free(inputpath);
+  curl_free(gatewayscheme);
+  curl_free(gatewayport);
   curl_free(cid);
   curl_free(pathbuffer);
-  curl_url_cleanup(ipfsurl);
+  curl_url_cleanup(gatewaysurl);
 
   switch(result) {
   case CURLE_URL_MALFORMAT:

tests/data/DISABLED

@@ -70,9 +70,6 @@
 266
 579
 587
-722
-724
-727
 # 1021 re-added here due to flakiness
 1021
 1117

tests/data/Makefile.inc

@@ -101,7 +101,8 @@ test681 test682 test683 test684 test685 test686 test687 test688 \
 test700 test701 test702 test703 test704 test705 test706 test707 test708 \
 test709 test710 test711 test712 test713 test714 test715 test716 test717 \
 test718 test719 test720 test721 test722 test723 test724 test725 test726 \
-test727 test728 test729 \
+test727 test728 test729 test730 test731 test732 test733 test734 test735 \
+test736 test737 test738 test739 test740 test741 \
 \
 test799 test800 test801 test802 test803 test804 test805 test806 test807 \
 test808 test809 test810 test811 test812 test813 test814 test815 test816 \

tests/data/test722

@@ -8,7 +8,7 @@ IPFS
 #
 # Server-side
 <reply>
-<data>
+<data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
@@ -34,7 +34,7 @@ http
 IPFS
 </name>
 <command>
---ipfs-gateway http://%HOSTIP:%HTTPPORT/%TESTNUMBER ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+--ipfs-gateway http://%HOSTIP:%HTTPPORT ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>
 
@@ -42,7 +42,7 @@ IPFS
 # Verify data after the test has been "shot"
 <verify>
 <protocol crlf="yes">
-GET /ipfs/QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx HTTP/1.1
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*

tests/data/test723

@@ -20,7 +20,7 @@ http
 IPFS with malformed gateway URL (bad function argument error)
 </name>
 <command>
---ipfs-gateway http://nonexisting,local:8080/%TESTNUMBER ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+--ipfs-gateway http://nonexisting,local:8080 ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>
 

tests/data/test724

@@ -8,7 +8,7 @@ IPFS
 #
 # Server-side
 <reply>
-<data>
+<data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
@@ -37,10 +37,10 @@ HOME=%PWD/%LOGDIR
 IPFS with gateway URL from gateway file
 </name>
 <command>
-ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 <file name="%LOGDIR/.ipfs/gateway" >
-http://%HOSTIP:%HTTPPORT/%TESTNUMBER
+http://%HOSTIP:%HTTPPORT
 </file>
 </client>
 
@@ -48,7 +48,7 @@ http://%HOSTIP:%HTTPPORT/%TESTNUMBER
 # Verify data after the test has been "shot"
 <verify>
 <protocol crlf="yes">
-GET /ipfs/QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx HTTP/1.1
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*

tests/data/test725

@@ -23,10 +23,10 @@ HOME=%PWD/%LOGDIR
 IPFS with malformed gateway URL from gateway file
 </name>
 <command>
-ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 <file name="%LOGDIR/.ipfs/gateway" >
-http://nonexisting,local:8080/%TESTNUMBER
+http://nonexisting,local:8080
 </file>
 </client>
 

tests/data/test726

@@ -26,7 +26,7 @@ HOME=%PWD
 IPFS with no gateway URL (no environment or home file either)
 </name>
 <command>
-ipfs://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>
 

tests/data/test727

@@ -8,7 +8,7 @@ IPNS
 #
 # Server-side
 <reply>
-<data>
+<data nocheck="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Server: test-server/fake
@@ -34,7 +34,7 @@ http
 IPNS
 </name>
 <command>
---ipfs-gateway http://%HOSTIP:%HTTPPORT/%TESTNUMBER ipns://QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx
+--ipfs-gateway http://%HOSTIP:%HTTPPORT ipns://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
 </command>
 </client>
 
@@ -42,7 +42,7 @@ IPNS
 # Verify data after the test has been "shot"
 <verify>
 <protocol crlf="yes">
-GET /ipns/QmV5JejrpgUxnkZeFZYMxVCqAbKy3KdPXWXyuEDiMNZwUx HTTP/1.1
+GET /ipns/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 User-Agent: curl/%VERSION
 Accept: */*

tests/data/test730

@@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS arg gateway with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT/foo/bar ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /foo/bar/ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test731

@@ -0,0 +1,58 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+</setenv>
+<name>
+IPFS with gateway URL and path from gateway file
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /%TESTNUMBER/ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test732

@@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT "ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test733

@@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS with path and query args
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT "ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test734

@@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPFS with path, query args and gateway with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT/some/path "ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /some/path/ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u/a/b?foo=bar&aaa=bbb HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test735

@@ -0,0 +1,52 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPNS with path, query args and gateway with path
+</name>
+<command>
+--ipfs-gateway http://%HOSTIP:%HTTPPORT/some/path "ipns://fancy.tld/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /some/path/ipns/fancy.tld/a/b?foo=bar&aaa=bbb HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test736

@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+IPFS_DATA=$HOME/.ipfs
+</setenv>
+<name>
+IPFS with IPFS_DATA set, no traling slash
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test737

@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+IPFS_DATA=$HOME/.ipfs/
+</setenv>
+<name>
+IPFS with IPFS_DATA set, with traling slash
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test738

@@ -0,0 +1,38 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+IPFS_DATA=%HOME/.ipfs/
+</setenv>
+<name>
+IPFS with IPFS_DATA, no gateway file
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+</client>
+
+#
+# Verify error code with no gateway file (detection fails)
+<verify>
+<errorcode>
+37
+</errorcode>
+</verify>
+</testcase>

tests/data/test739

@@ -0,0 +1,34 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+IPNS path and query args for gateway and IPFS url (malformed gateway url)
+</name>
+<command>
+--ipfs-gateway "http://%HOSTIP:%HTTPPORT/some/path?biz=baz" "ipns://fancy.tld/a/b?foo=bar&aaa=bbb"
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+3
+</errorcode>
+</verify>
+</testcase>

tests/data/test740

@@ -0,0 +1,60 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 21
+Connection: close
+Content-Type: text/plain
+Funny-head: yesyes
+
+Hello curl from IPFS
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+</setenv>
+<name>
+IPFS with gateway URL from multiline gateway file
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+http://%HOSTIP:%HTTPPORT
+foo
+bar
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /ipfs/bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>

tests/data/test741

@@ -0,0 +1,42 @@
+<testcase>
+<info>
+<keywords>
+IPFS
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<setenv>
+HOME=%PWD/%LOGDIR
+</setenv>
+<name>
+IPFS with malformed gateway URL from multiline gateway file, first line no url
+</name>
+<command>
+ipfs://bafybeidecnvkrygux6uoukouzps5ofkeevoqland7kopseiod6pzqvjg7u
+</command>
+<file name="%LOGDIR/.ipfs/gateway" >
+foo
+bar
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+# malformed gateway URL, first line in file must be a gateway URL
+<errorcode>
+3
+</errorcode>
+</verify>
+</testcase>