tool_writeout: add URL component variables

Output specific components from the used URL. The following variables are added for this purpose: url.scheme, url.user, url.password, url.options, url.host, url.port, url.path, url.query, url.fragment, url.zoneid Add the following for outputting parts of the "effective URL": urle.scheme, urle.user, urle.password, urle.options, urle.host, urle.port, urle.path, urle.query, urle.fragment, urle.zoneid Added test 423 and 424 to verify. Closes #10853
2023-04-04 14:42:44 +02:00 · 2023-04-04 14:42:44 +02:00 · 808cb31756
commit 808cb31756
parent e0c3424fb1
8 changed files with 337 additions and 3 deletions
--- a/docs/cmdline-opts/write-out.d
+++ b/docs/cmdline-opts/write-out.d
@ -209,6 +209,70 @@ The total time, in seconds, that the full operation lasted.
 .B url
 The URL that was fetched. (Added in 7.75.0)
 .TP
+.B url.scheme
+The scheme part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.user
+The user part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.password
+The password part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.options
+The options part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.host
+The host part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.port
+The port number of the URL that was fetched. If no port number was specified,
+but the URL scheme is known, that scheme's default port number is
+shown. (Added in 8.1.0)
+.TP
+.B url.path
+The path part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.query
+The query part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.fragment
+The fragment part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.zoneid
+The zoneid part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.scheme
+The scheme part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.user
+The user part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.password
+The password part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.options
+The options part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.host
+The host part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.port
+The port number of the effective (last) URL that was fetched. If no port
+number was specified, but the URL scheme is known, that scheme's default port
+number is shown. (Added in 8.1.0)
+.TP
+.B urle.path
+The path part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.query
+The query part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.fragment
+The fragment part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.zoneid
+The zoneid part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
 .B urlnum
 The URL index number of this transfer, 0-indexed. De-globbed URLs share the
 same index number as the origin globbed URL. (Added in 7.75.0)
--- a/src/tool_writeout.c
+++ b/src/tool_writeout.c
@ -123,6 +123,26 @@ static const struct writeoutvar variables[] = {
   writeTime},
  {"time_total", VAR_TOTAL_TIME, CURLINFO_TOTAL_TIME_T, writeTime},
  {"url", VAR_INPUT_URL, CURLINFO_NONE, writeString},
+  {"url.scheme", VAR_INPUT_URLSCHEME, CURLINFO_NONE, writeString},
+  {"url.user", VAR_INPUT_URLUSER, CURLINFO_NONE, writeString},
+  {"url.password", VAR_INPUT_URLPASSWORD, CURLINFO_NONE, writeString},
+  {"url.options", VAR_INPUT_URLOPTIONS, CURLINFO_NONE, writeString},
+  {"url.host", VAR_INPUT_URLHOST, CURLINFO_NONE, writeString},
+  {"url.port", VAR_INPUT_URLPORT, CURLINFO_NONE, writeString},
+  {"url.path", VAR_INPUT_URLPATH, CURLINFO_NONE, writeString},
+  {"url.query", VAR_INPUT_URLQUERY, CURLINFO_NONE, writeString},
+  {"url.fragment", VAR_INPUT_URLFRAGMENT, CURLINFO_NONE, writeString},
+  {"url.zoneid", VAR_INPUT_URLZONEID, CURLINFO_NONE, writeString},
+  {"urle.scheme", VAR_INPUT_URLESCHEME, CURLINFO_NONE, writeString},
+  {"urle.user", VAR_INPUT_URLEUSER, CURLINFO_NONE, writeString},
+  {"urle.password", VAR_INPUT_URLEPASSWORD, CURLINFO_NONE, writeString},
+  {"urle.options", VAR_INPUT_URLEOPTIONS, CURLINFO_NONE, writeString},
+  {"urle.host", VAR_INPUT_URLEHOST, CURLINFO_NONE, writeString},
+  {"urle.port", VAR_INPUT_URLEPORT, CURLINFO_NONE, writeString},
+  {"urle.path", VAR_INPUT_URLEPATH, CURLINFO_NONE, writeString},
+  {"urle.query", VAR_INPUT_URLEQUERY, CURLINFO_NONE, writeString},
+  {"urle.fragment", VAR_INPUT_URLEFRAGMENT, CURLINFO_NONE, writeString},
+  {"urle.zoneid", VAR_INPUT_URLEZONEID, CURLINFO_NONE, writeString},
  {"url_effective", VAR_EFFECTIVE_URL, CURLINFO_EFFECTIVE_URL, writeString},
  {"urlnum", VAR_URLNUM, CURLINFO_NONE, writeLong},
  {NULL, VAR_NONE, CURLINFO_NONE, NULL}
@ -165,12 +185,94 @@ static int writeTime(FILE *stream, const struct writeoutvar *wovar,
  return 1; /* return 1 if anything was written */
 }

+static int urlpart(struct per_transfer *per, writeoutid vid,
+                   const char **contentp)
+{
+  CURLU *uh = curl_url();
+  int rc = 0;
+  if(uh) {
+    CURLUPart cpart = CURLUPART_HOST;
+    char *part = NULL;
+    const char *url = NULL;
+
+    if(vid >= VAR_INPUT_URLEHOST) {
+      if(curl_easy_getinfo(per->curl, CURLINFO_EFFECTIVE_URL, &url))
+        rc = 5;
+    }
+    else
+      url = per->this_url;
+
+    if(!rc) {
+      switch(vid) {
+      case VAR_INPUT_URLSCHEME:
+      case VAR_INPUT_URLESCHEME:
+        cpart = CURLUPART_SCHEME;
+        break;
+      case VAR_INPUT_URLUSER:
+      case VAR_INPUT_URLEUSER:
+        cpart = CURLUPART_USER;
+        break;
+      case VAR_INPUT_URLPASSWORD:
+      case VAR_INPUT_URLEPASSWORD:
+        cpart = CURLUPART_PASSWORD;
+        break;
+      case VAR_INPUT_URLOPTIONS:
+      case VAR_INPUT_URLEOPTIONS:
+        cpart = CURLUPART_OPTIONS;
+        break;
+      case VAR_INPUT_URLHOST:
+      case VAR_INPUT_URLEHOST:
+        cpart = CURLUPART_HOST;
+        break;
+      case VAR_INPUT_URLPORT:
+      case VAR_INPUT_URLEPORT:
+        cpart = CURLUPART_PORT;
+        break;
+      case VAR_INPUT_URLPATH:
+      case VAR_INPUT_URLEPATH:
+        cpart = CURLUPART_PATH;
+        break;
+      case VAR_INPUT_URLQUERY:
+      case VAR_INPUT_URLEQUERY:
+        cpart = CURLUPART_QUERY;
+        break;
+      case VAR_INPUT_URLFRAGMENT:
+      case VAR_INPUT_URLEFRAGMENT:
+        cpart = CURLUPART_FRAGMENT;
+        break;
+      case VAR_INPUT_URLZONEID:
+      case VAR_INPUT_URLEZONEID:
+        cpart = CURLUPART_ZONEID;
+        break;
+      default:
+        /* not implemented */
+        rc = 4;
+        break;
+      }
+    }
+    if(!rc && curl_url_set(uh, CURLUPART_URL, url,
+                           CURLU_GUESS_SCHEME|CURLU_NON_SUPPORT_SCHEME))
+      rc = 2;
+
+    if(!rc && curl_url_get(uh, cpart, &part, CURLU_DEFAULT_PORT))
+      rc = 3;
+
+    if(!rc && part)
+      *contentp = part;
+    curl_url_cleanup(uh);
+  }
+  else
+    return 1;
+  return rc;
+}
+
 static int writeString(FILE *stream, const struct writeoutvar *wovar,
                       struct per_transfer *per, CURLcode per_result,
                       bool use_json)
 {
  bool valid = false;
  const char *strinfo = NULL;
+  const char *freestr = NULL;
  struct dynbuf buf;
  curlx_dyn_init(&buf, 256*1024);

@ -262,6 +364,33 @@ static int writeString(FILE *stream, const struct writeoutvar *wovar,
        valid = true;
      }
      break;
+    case VAR_INPUT_URLSCHEME:
+    case VAR_INPUT_URLUSER:
+    case VAR_INPUT_URLPASSWORD:
+    case VAR_INPUT_URLOPTIONS:
+    case VAR_INPUT_URLHOST:
+    case VAR_INPUT_URLPORT:
+    case VAR_INPUT_URLPATH:
+    case VAR_INPUT_URLQUERY:
+    case VAR_INPUT_URLFRAGMENT:
+    case VAR_INPUT_URLZONEID:
+    case VAR_INPUT_URLESCHEME:
+    case VAR_INPUT_URLEUSER:
+    case VAR_INPUT_URLEPASSWORD:
+    case VAR_INPUT_URLEOPTIONS:
+    case VAR_INPUT_URLEHOST:
+    case VAR_INPUT_URLEPORT:
+    case VAR_INPUT_URLEPATH:
+    case VAR_INPUT_URLEQUERY:
+    case VAR_INPUT_URLEFRAGMENT:
+    case VAR_INPUT_URLEZONEID:
+      if(per->this_url) {
+        if(!urlpart(per, wovar->id, &strinfo)) {
+          freestr = strinfo;
+          valid = true;
+        }
+      }
+      break;
    default:
      DEBUGASSERT(0);
      break;
@ -281,6 +410,7 @@ static int writeString(FILE *stream, const struct writeoutvar *wovar,
    if(use_json)
      fprintf(stream, "\"%s\":null", wovar->name);
  }
+  curl_free((char *)freestr);

  curlx_dyn_free(&buf);
  return 1; /* return 1 if anything was written */
--- a/src/tool_writeout.h
+++ b/src/tool_writeout.h
@ -44,6 +44,27 @@ typedef enum {
  VAR_HTTP_CODE_PROXY,
  VAR_HTTP_VERSION,
  VAR_INPUT_URL,
+  VAR_INPUT_URLSCHEME,
+  VAR_INPUT_URLUSER,
+  VAR_INPUT_URLPASSWORD,
+  VAR_INPUT_URLOPTIONS,
+  VAR_INPUT_URLHOST,
+  VAR_INPUT_URLPORT,
+  VAR_INPUT_URLPATH,
+  VAR_INPUT_URLQUERY,
+  VAR_INPUT_URLFRAGMENT,
+  VAR_INPUT_URLZONEID,
+  /* the same ones again for url *effective* */
+  VAR_INPUT_URLESCHEME,
+  VAR_INPUT_URLEUSER,
+  VAR_INPUT_URLEPASSWORD,
+  VAR_INPUT_URLEOPTIONS,
+  VAR_INPUT_URLEHOST,
+  VAR_INPUT_URLEPORT,
+  VAR_INPUT_URLEPATH,
+  VAR_INPUT_URLEQUERY,
+  VAR_INPUT_URLEFRAGMENT,
+  VAR_INPUT_URLEZONEID,
  VAR_JSON,
  VAR_LOCAL_IP,
  VAR_LOCAL_PORT,
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@ -68,7 +68,7 @@ test380 test381 test383 test384 test385 test386 test387 test388 test389 \
 test390 test391 test392 test393 test394 test395 test396 test397 test398 \
 test399 test400 test401 test402 test403 test404 test405 test406 test407 \
 test408 test409 test410 test411 test412 test413 test414 test415 test416 \
-test417 test418 test419 test420 test421 test422 \
+test417 test418 test419 test420 test421 test422 test423 test424 \
 \
 test430 test431 test432 test433 test434 test435 test436 \
 \
--- a/tests/data/test423
+++ b/tests/data/test423
@ -0,0 +1,51 @@
+<testcase>
+<info>
+<keywords>
+-w
+--write-out
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 0
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+-w with url.* variables
+</name>
+<command option="no-include">
+"http://uuuu:pppp@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" "h55p://hello2000:1/%TESTNUMBER?qqqq#ffff" "local host" "http://u22u:p22p@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" -w '%{url.host}+%{url.path}+%{url.scheme}+%{url.user}+%{url.password}+%{url.port}+%{url.query}+%{url.fragment}\n'
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stdout>
+%HOSTIP+/%TESTNUMBER+http+uuuu+pppp+%HTTPPORT+qqqq+ffff
+hello2000+/%TESTNUMBER+h55p+++1+qqqq+ffff
+++++++
+%HOSTIP+/%TESTNUMBER+http+u22u+p22p+%HTTPPORT+qqqq+ffff
+</stdout>
+</verify>
+</testcase>
--- a/tests/data/test424
+++ b/tests/data/test424
@ -0,0 +1,68 @@
+<testcase>
+<info>
+<keywords>
+-w
+--write-out
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 301 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 0
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+Location: http://anotherhost.example:2023/%TESTNUMBER0002?moo.html
+
+</data>
+<data2 crlf="yes" nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 0
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+</data2>
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+proxy
+</features>
+<server>
+http
+</server>
+<name>
+-w with urle.* variables
+</name>
+<command option="no-include">
+"http://uuuu:pppp@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" "h55p://hello2000:1/%TESTNUMBER?qqqq#ffff" "local host" "http://u22u:p22p@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" -w '%{urle.host}+%{urle.path}+%{urle.scheme}+%{urle.user}+%{urle.password}+%{urle.port}+%{urle.query}+%{urle.fragment}\n' -x http://%HOSTIP:%HTTPPORT/ -L
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stdout>
+anotherhost.example+/%TESTNUMBER0002+http+uuuu+pppp+2023+moo.html+
+hello2000+/%TESTNUMBER+h55p+++1+qqqq+ffff
+++++++
+anotherhost.example+/%TESTNUMBER0002+http+u22u+p22p+2023+moo.html+
+</stdout>
+</verify>
+</testcase>
--- a/tests/data/test970
+++ b/tests/data/test970
@ -59,7 +59,7 @@ Accept: */*

 </protocol>
 <stdout nonewline="yes">
-{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
+{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.scheme":"http","url.user":null,"url.password":null,"url.options":null,"url.host":"%HOSTIP","url.port":"%HTTPPORT","url.path":"/%TESTNUMBER","url.query":null,"url.fragment":null,"url.zoneid":null,"urle.scheme":"http","urle.user":null,"urle.password":null,"urle.options":null,"urle.host":"%HOSTIP","urle.port":"%HTTPPORT","urle.path":"/%TESTNUMBER","urle.query":null,"urle.fragment":null,"urle.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>
--- a/tests/data/test972
+++ b/tests/data/test972
@ -60,7 +60,7 @@ Accept: */*

 </protocol>
 <stdout mode="text">
-{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out972","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"%HOSTIP","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
+{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out972","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"%HOSTIP","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.scheme":"http","url.user":null,"url.password":null,"url.options":null,"url.host":"%HOSTIP","url.port":"%HTTPPORT","url.path":"/%TESTNUMBER","url.query":null,"url.fragment":null,"url.zoneid":null,"urle.scheme":"http","urle.user":null,"urle.password":null,"urle.options":null,"urle.host":"%HOSTIP","urle.port":"%HTTPPORT","urle.path":"/%TESTNUMBER","urle.query":null,"urle.fragment":null,"urle.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>