curl: move IPFS code into src/tool_ipfs.[ch]

- convert ensure_trailing into ensure_trailing_slash
- strdup the URL string to own it proper
- use shorter variable names
- combine some expressions
- simplify error handling in ipfs_gateway()
- add MAX_GATEWAY_URL_LEN + proper bailout if maximum is reached
- ipfs-gateway.d polish and simplification
- shorten ipfs error message + make them "synthetic"

Closes #12281
This commit is contained in:
Daniel Stenberg 2023-11-06 13:24:43 +01:00
parent fd7ef00f43
commit 01d9b8bc7c
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
5 changed files with 346 additions and 317 deletions

View File

@ -9,36 +9,24 @@ Category: ipfs
Example: --ipfs-gateway $URL ipfs://
Multi: single
---
Specifies which gateway to use for IPFS and IPNS URLs.
Not specifying this argument will let cURL try to automatically
check if IPFS_GATEWAY environment variable is set,
or if ~/.ipfs/gateway plain text file exists.
Specify which gateway to use for IPFS and IPNS URLs. Not specifying this will
instead make curl check if the IPFS_GATEWAY environment variable is set, or if
a ~/.ipfs/gateway file holding the gateway URL exists.
If you run a local IPFS node, this gateway is by default
available under http://localhost:8080. A full example URL would
look like:
If you run a local IPFS node, this gateway is by default available under
http://localhost:8080. A full example URL would look like:
curl --ipfs-gateway http://localhost:8080 ipfs://bafybeigagd5nmnn2iys2f3doro7ydrevyr2mzarwidgadawmamiteydbzi
You can also specify publicly available gateways. One such
gateway is https://ipfs.io. A full example url would look like:
curl --ipfs-gateway https://ipfs.io ipfs://bafybeigagd5nmnn2iys2f3doro7ydrevyr2mzarwidgadawmamiteydbzi
There are many public IPFS gateways. As a starting point to find
one that works for your case, consult this page:
There are many public IPFS gateways. See for example:
https://ipfs.github.io/public-gateway-checker/
A word of caution! When you opt to go for a remote gateway you should
be aware that you completely trust the gateway. This is fine in local gateways
as you host it yourself. With remote gateways there could potentially be
a malicious actor returning you data that does not match the request you made,
inspect or even interfere with the request. You won't notice this when using cURL.
A mitigation could be to go for a "trustless" gateway. This means you
locally verify that the data. Consult the docs page on trusted vs trustless:
WARNING: If you opt to go for a remote gateway you should be aware that you
completely trust the gateway. This is fine in local gateways as you host it
yourself. With remote gateways there could potentially be a malicious actor
returning you data that does not match the request you made, inspect or even
interfere with the request. You will not notice this when using curl. A
mitigation could be to go for a "trustless" gateway. This means you locally
verify that the data. Consult the docs page on trusted vs trustless:
https://docs.ipfs.tech/reference/http/gateway/#trusted-vs-trustless

View File

@ -79,6 +79,7 @@ CURL_CFILES = \
tool_help.c \
tool_helpers.c \
tool_hugehelp.c \
tool_ipfs.c \
tool_libinfo.c \
tool_listhelp.c \
tool_main.c \
@ -122,6 +123,7 @@ CURL_HFILES = \
tool_help.h \
tool_helpers.h \
tool_hugehelp.h \
tool_ipfs.h \
tool_libinfo.h \
tool_main.h \
tool_msgs.h \

295
src/tool_ipfs.c Normal file
View File

@ -0,0 +1,295 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
* SPDX-License-Identifier: curl
*
***************************************************************************/
#include "tool_setup.h"
#define ENABLE_CURLX_PRINTF
/* use our own printf() functions */
#include "curlx.h"
#include "dynbuf.h"
#include "tool_cfgable.h"
#include "tool_msgs.h"
#include "tool_ipfs.h"
#include "memdebug.h" /* keep this as LAST include */
/* ensure input ends in slash */
static CURLcode ensure_trailing_slash(char **input)
{
if(*input && **input) {
size_t len = strlen(*input);
if(((*input)[len - 1] != '/')) {
struct curlx_dynbuf dyn;
curlx_dyn_init(&dyn, len + 2);
if(curlx_dyn_addn(&dyn, *input, len)) {
Curl_safefree(*input);
return CURLE_OUT_OF_MEMORY;
}
Curl_safefree(*input);
if(curlx_dyn_addn(&dyn, "/", 1))
return CURLE_OUT_OF_MEMORY;
*input = curlx_dyn_ptr(&dyn);
}
}
return CURLE_OK;
}
static char *ipfs_gateway(void)
{
char *ipfs_path = NULL;
char *gateway_composed_file_path = NULL;
FILE *gateway_file = NULL;
char *gateway = curlx_getenv("IPFS_GATEWAY");
/* Gateway is found from environment variable. */
if(gateway) {
if(ensure_trailing_slash(&gateway))
goto fail;
return gateway;
}
/* Try to find the gateway in the IPFS data folder. */
ipfs_path = curlx_getenv("IPFS_PATH");
if(!ipfs_path) {
char *home = curlx_getenv("HOME");
if(home && *home)
ipfs_path = aprintf("%s/.ipfs/", home);
/* fallback to "~/.ipfs", as that's the default location. */
Curl_safefree(home);
}
if(!ipfs_path || ensure_trailing_slash(&ipfs_path))
goto fail;
gateway_composed_file_path = aprintf("%sgateway", ipfs_path);
if(!gateway_composed_file_path)
goto fail;
gateway_file = fopen(gateway_composed_file_path, FOPEN_READTEXT);
Curl_safefree(gateway_composed_file_path);
if(gateway_file) {
int c;
struct curlx_dynbuf dyn;
curlx_dyn_init(&dyn, MAX_GATEWAY_URL_LEN);
/* get the first line of the gateway file, ignore the rest */
while((c = getc(gateway_file)) != EOF && c != '\n' && c != '\r') {
if(curlx_dyn_addn(&dyn, &c, 1))
goto fail;
}
fclose(gateway_file);
gateway_file = NULL;
if(curlx_dyn_len(&dyn))
gateway = curlx_dyn_ptr(&dyn);
if(gateway)
ensure_trailing_slash(&gateway);
if(!gateway)
goto fail;
Curl_safefree(ipfs_path);
return gateway;
}
fail:
if(gateway_file)
fclose(gateway_file);
Curl_safefree(gateway);
Curl_safefree(ipfs_path);
return NULL;
}
/*
* Rewrite ipfs://<cid> and ipns://<cid> to a HTTP(S)
* URL that can be handled by an IPFS gateway.
*/
CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
struct OperationConfig *config)
{
CURLcode result = CURLE_URL_MALFORMAT;
CURLUcode getResult;
char *gateway = NULL;
char *gwhost = NULL;
char *gwpath = NULL;
char *gwquery = NULL;
char *gwscheme = NULL;
char *gwport = NULL;
char *inputpath = NULL;
char *cid = NULL;
char *pathbuffer = NULL;
char *cloneurl;
CURLU *gatewayurl = curl_url();
if(!gatewayurl) {
result = CURLE_FAILED_INIT;
goto clean;
}
getResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE);
if(getResult || !cid)
goto clean;
/* We might have a --ipfs-gateway argument. Check it first and use it. Error
* if we do have something but if it's an invalid url.
*/
if(config->ipfs_gateway) {
/* ensure the gateway ends in a trailing / */
if(ensure_trailing_slash(&config->ipfs_gateway) != CURLE_OK) {
result = CURLE_OUT_OF_MEMORY;
goto clean;
}
if(!curl_url_set(gatewayurl, CURLUPART_URL, config->ipfs_gateway,
CURLU_GUESS_SCHEME)) {
gateway = strdup(config->ipfs_gateway);
if(!gateway) {
result = CURLE_URL_MALFORMAT;
goto clean;
}
}
else {
result = CURLE_BAD_FUNCTION_ARGUMENT;
goto clean;
}
}
else {
/* this is ensured to end in a trailing / within ipfs_gateway() */
gateway = ipfs_gateway();
if(!gateway) {
result = CURLE_FILE_COULDNT_READ_FILE;
goto clean;
}
if(curl_url_set(gatewayurl, CURLUPART_URL, gateway, 0)) {
result = CURLE_URL_MALFORMAT;
goto clean;
}
}
/* check for unsupported gateway parts */
if(curl_url_get(gatewayurl, CURLUPART_QUERY, &gwquery, 0)
!= CURLUE_NO_QUERY) {
result = CURLE_URL_MALFORMAT;
goto clean;
}
/* get gateway parts */
if(curl_url_get(gatewayurl, CURLUPART_HOST,
&gwhost, CURLU_URLDECODE)) {
goto clean;
}
if(curl_url_get(gatewayurl, CURLUPART_SCHEME,
&gwscheme, CURLU_URLDECODE)) {
goto clean;
}
curl_url_get(gatewayurl, CURLUPART_PORT, &gwport, CURLU_URLDECODE);
curl_url_get(gatewayurl, CURLUPART_PATH, &gwpath, CURLU_URLDECODE);
/* get the path from user input */
curl_url_get(uh, CURLUPART_PATH, &inputpath, CURLU_URLDECODE);
/* inputpath might be NULL or a valid pointer now */
/* set gateway parts in input url */
if(curl_url_set(uh, CURLUPART_SCHEME, gwscheme, CURLU_URLENCODE) ||
curl_url_set(uh, CURLUPART_HOST, gwhost, CURLU_URLENCODE) ||
curl_url_set(uh, CURLUPART_PORT, gwport, CURLU_URLENCODE))
goto clean;
/* if the input path is just a slash, clear it */
if(inputpath && (inputpath[0] == '/') && !inputpath[1])
*inputpath = '\0';
/* ensure the gateway path ends with a trailing slash */
ensure_trailing_slash(&gwpath);
pathbuffer = aprintf("%s%s/%s%s", gwpath, protocol, cid,
inputpath ? inputpath : "");
if(!pathbuffer) {
goto clean;
}
if(curl_url_set(uh, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)) {
goto clean;
}
/* Free whatever it has now, rewriting is next */
Curl_safefree(*url);
if(curl_url_get(uh, CURLUPART_URL, &cloneurl, CURLU_URLENCODE)) {
goto clean;
}
/* we need to strdup the URL so that we can call free() on it later */
*url = strdup(cloneurl);
curl_free(cloneurl);
if(!*url)
goto clean;
result = CURLE_OK;
clean:
free(gateway);
curl_free(gwhost);
curl_free(gwpath);
curl_free(gwquery);
curl_free(inputpath);
curl_free(gwscheme);
curl_free(gwport);
curl_free(cid);
curl_free(pathbuffer);
curl_url_cleanup(gatewayurl);
{
const char *msg = NULL;
switch(result) {
case CURLE_URL_MALFORMAT:
msg = "malformed target URL";
break;
case CURLE_FILE_COULDNT_READ_FILE:
msg = "IPFS automatic gateway detection failed";
break;
case CURLE_BAD_FUNCTION_ARGUMENT:
msg = "--ipfs-gateway was given a malformed URL";
break;
default:
break;
}
if(msg)
helpf(tool_stderr, msg);
}
return result;
}

33
src/tool_ipfs.h Normal file
View File

@ -0,0 +1,33 @@
#ifndef HEADER_CURL_TOOL_IPFS_H
#define HEADER_CURL_TOOL_IPFS_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
* SPDX-License-Identifier: curl
*
***************************************************************************/
#include "tool_setup.h"
#define MAX_GATEWAY_URL_LEN 10000
CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
struct OperationConfig *config);
#endif /* HEADER_CURL_TOOL_IPFS_H */

View File

@ -81,6 +81,7 @@
#include "tool_help.h"
#include "tool_hugehelp.h"
#include "tool_progress.h"
#include "tool_ipfs.h"
#include "dynbuf.h"
#include "memdebug.h" /* keep this as LAST include */
@ -697,296 +698,6 @@ noretry:
return result;
}
/* helper function to ensure input ends in val_to_ensure */
static CURLcode ensure_trailing(char **input, const char val_to_ensure)
{
if(*input && **input) {
size_t len = strlen(*input);
if(((*input)[len - 1] != val_to_ensure)) {
struct curlx_dynbuf dyn;
curlx_dyn_init(&dyn, len + 2);
if(curlx_dyn_addn(&dyn, *input, len)) {
Curl_safefree(*input);
return CURLE_OUT_OF_MEMORY;
}
Curl_safefree(*input);
if(curlx_dyn_addn(&dyn, &val_to_ensure, 1))
return CURLE_OUT_OF_MEMORY;
*input = curlx_dyn_ptr(&dyn);
}
}
return CURLE_OK;
}
static char *ipfs_gateway(void)
{
char *gateway = NULL;
char *ipfs_path = NULL;
char *gateway_composed_file_path = NULL;
FILE *gateway_file = NULL;
gateway = curlx_getenv("IPFS_GATEWAY");
/* Gateway is found from environment variable. */
if(gateway) {
if(ensure_trailing(&gateway, '/')) {
Curl_safefree(gateway);
return NULL;
}
return gateway;
}
/* Try to find the gateway in the IPFS data folder. */
ipfs_path = curlx_getenv("IPFS_PATH");
if(!ipfs_path) {
char *home = curlx_getenv("HOME");
if(home && *home)
ipfs_path = aprintf("%s/.ipfs/", home);
/* fallback to "~/.ipfs", as that's the default location. */
Curl_safefree(home);
}
if(!ipfs_path) {
Curl_safefree(gateway);
Curl_safefree(ipfs_path);
return NULL;
}
else {
if(ensure_trailing(&ipfs_path, '/')) {
Curl_safefree(gateway);
Curl_safefree(ipfs_path);
return NULL;
}
}
gateway_composed_file_path = aprintf("%sgateway", ipfs_path);
if(!gateway_composed_file_path) {
Curl_safefree(gateway);
Curl_safefree(ipfs_path);
return NULL;
}
gateway_file = fopen(gateway_composed_file_path, FOPEN_READTEXT);
Curl_safefree(gateway_composed_file_path);
if(gateway_file) {
int c;
struct curlx_dynbuf dyn;
curlx_dyn_init(&dyn, INT_MAX);
/* get the first line of the gateway file, ignore the rest */
while((c = getc(gateway_file)) != EOF && c != '\n' && c != '\r') {
if(curlx_dyn_addn(&dyn, &c, 1))
break;
}
if(gateway_file)
fclose(gateway_file);
if(curlx_dyn_len(&dyn) > 0)
gateway = curlx_dyn_ptr(&dyn);
if(gateway)
ensure_trailing(&gateway, '/');
if(!gateway) {
Curl_safefree(ipfs_path);
return NULL;
}
Curl_safefree(ipfs_path);
return gateway;
}
Curl_safefree(gateway);
Curl_safefree(ipfs_path);
return NULL;
}
/*
* Rewrite ipfs://<cid> and ipns://<cid> to a HTTP(S)
* URL that can be handled by an IPFS gateway.
*/
static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url,
struct OperationConfig *config)
{
CURLcode result = CURLE_URL_MALFORMAT;
CURLUcode getResult;
char *gateway = NULL;
char *gatewayhost = NULL;
char *gatewaypath = NULL;
char *gatewayquery = NULL;
char *inputpath = NULL;
char *gatewayscheme = NULL;
char *gatewayport = NULL;
char *cid = NULL;
char *pathbuffer = NULL;
CURLU *gatewaysurl = curl_url();
if(!gatewaysurl) {
result = CURLE_FAILED_INIT;
goto clean;
}
getResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE);
if(getResult) {
goto clean;
}
if(!cid) {
goto clean;
}
/* We might have a --ipfs-gateway argument. Check it first and use it. Error
* if we do have something but if it's an invalid url.
*/
if(config->ipfs_gateway) {
/* ensure the gateway ends in a trailing / */
if(ensure_trailing(&config->ipfs_gateway, '/') != CURLE_OK) {
result = CURLE_OUT_OF_MEMORY;
goto clean;
}
if(!curl_url_set(gatewaysurl, CURLUPART_URL, config->ipfs_gateway,
CURLU_GUESS_SCHEME)) {
gateway = strdup(config->ipfs_gateway);
if(!gateway) {
result = CURLE_URL_MALFORMAT;
goto clean;
}
}
else {
result = CURLE_BAD_FUNCTION_ARGUMENT;
goto clean;
}
}
else {
/* this is ensured to end in a trailing / within ipfs_gateway() */
gateway = ipfs_gateway();
if(!gateway) {
result = CURLE_FILE_COULDNT_READ_FILE;
goto clean;
}
if(curl_url_set(gatewaysurl, CURLUPART_URL, gateway, 0)) {
result = CURLE_URL_MALFORMAT;
goto clean;
}
}
/* check for unsupported gateway parts */
if(curl_url_get(gatewaysurl, CURLUPART_QUERY, &gatewayquery, 0)
!= CURLUE_NO_QUERY) {
result = CURLE_URL_MALFORMAT;
goto clean;
}
/* get gateway parts */
if(curl_url_get(gatewaysurl, CURLUPART_HOST,
&gatewayhost, CURLU_URLDECODE)) {
goto clean;
}
if(curl_url_get(gatewaysurl, CURLUPART_SCHEME,
&gatewayscheme, CURLU_URLDECODE)) {
goto clean;
}
curl_url_get(gatewaysurl, CURLUPART_PORT, &gatewayport, CURLU_URLDECODE);
curl_url_get(gatewaysurl, CURLUPART_PATH, &gatewaypath, CURLU_URLDECODE);
/* get the path from user input */
curl_url_get(uh, CURLUPART_PATH, &inputpath, CURLU_URLDECODE);
/* inputpath might be NULL or a valid pointer now */
/* set gateway parts in input url */
if(curl_url_set(uh, CURLUPART_SCHEME, gatewayscheme, CURLU_URLENCODE)) {
goto clean;
}
if(curl_url_set(uh, CURLUPART_HOST, gatewayhost, CURLU_URLENCODE)) {
goto clean;
}
if(curl_url_set(uh, CURLUPART_PORT, gatewayport, CURLU_URLENCODE)) {
goto clean;
}
/* if the input path is just a slash, clear it */
if(inputpath && (inputpath[0] == '/') && !inputpath[1])
*inputpath = '\0';
/* ensure the gateway path ends with a trailing slash */
ensure_trailing(&gatewaypath, '/');
pathbuffer = aprintf("%s%s/%s%s", gatewaypath, protocol, cid,
inputpath ? inputpath : "");
if(!pathbuffer) {
goto clean;
}
if(curl_url_set(uh, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)) {
goto clean;
}
/* Free whatever it has now, rewriting is next */
Curl_safefree(*url);
if(curl_url_get(uh, CURLUPART_URL, url, CURLU_URLENCODE)) {
goto clean;
}
result = CURLE_OK;
clean:
free(gateway);
curl_free(gatewayhost);
curl_free(gatewaypath);
curl_free(gatewayquery);
curl_free(inputpath);
curl_free(gatewayscheme);
curl_free(gatewayport);
curl_free(cid);
curl_free(pathbuffer);
curl_url_cleanup(gatewaysurl);
switch(result) {
case CURLE_URL_MALFORMAT:
helpf(tool_stderr, "malformed URL. Visit https://curl.se/"
"docs/ipfs.html#gateway-file-and-"
"environment-variable for more "
"information");
break;
case CURLE_FILE_COULDNT_READ_FILE:
helpf(tool_stderr, "IPFS automatic gateway detection "
"failure. Visit https://curl.se/docs/"
"ipfs.html#malformed-gateway-url for "
"more information");
break;
case CURLE_BAD_FUNCTION_ARGUMENT:
helpf(tool_stderr, "--ipfs-gateway argument results in "
"malformed URL. Visit https://curl.se/"
"docs/ipfs.html#malformed-gateway-url "
"for more information");
break;
default:
break;
}
return result;
}
/*
* Return the protocol token for the scheme used in the given URL
*/
@ -1010,13 +721,13 @@ static CURLcode url_proto(char **url,
if(curl_strequal(schemep, proto_ipfs) ||
curl_strequal(schemep, proto_ipns)) {
result = ipfs_url_rewrite(uh, schemep, url, config);
/* short-circuit proto_token, we know it's ipfs or ipns */
if(curl_strequal(schemep, proto_ipfs))
proto = proto_ipfs;
else if(curl_strequal(schemep, proto_ipns))
proto = proto_ipns;
if(result)
config->synthetic_error = TRUE;
}
else
proto = proto_token(schemep);