Interface for downloading HTTP, FTP documents. More...

#include "cpl_conv.h"
#include "cpl_string.h"
#include "cpl_progress.h"
#include "cpl_vsi.h"

Classes
struct	CPLMimePart

struct	CPLHTTPResult

class	GOA2Manager
	Manager of Google OAuth2 authentication. More...

Typedefs
typedef CPLHTTPResult (	CPLHTTPFetchCallbackFunc) (const char pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void pWriteArg, void pUserData)
	Callback function to process network requests. More...

Functions
int	CPLHTTPEnabled (void)
	Return if CPLHTTP services can be useful. More...

CPLHTTPResult *	CPLHTTPFetch (const char *pszURL, CSLConstList papszOptions)
	Fetch a document from an url and return in a string. More...

CPLHTTPResult *	CPLHTTPFetchEx (const char pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
	Fetch a document from an url and return in a string. More...

CPLHTTPResult **	CPLHTTPMultiFetch (const char const papszURL, int nURLCount, int nMaxSimultaneous, CSLConstList papszOptions)
	Fetch several documents at once. More...

void	CPLHTTPCleanup (void)
	Cleanup function to call at application termination.

void	CPLHTTPDestroyResult (CPLHTTPResult *psResult)
	Clean the memory associated with the return value of CPLHTTPFetch() More...

void	CPLHTTPDestroyMultiResult (CPLHTTPResult **papsResults, int nCount)
	Clean the memory associated with the return value of CPLHTTPMultiFetch() More...

int	CPLHTTPParseMultipartMime (CPLHTTPResult *psResult)
	Parses a MIME multipart message. More...

void	CPLHTTPSetDefaultUserAgent (const char *pszUserAgent)
	Set the default user agent. More...

void	CPLHTTPSetFetchCallback (CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
	Installs an alternate callback to the default implementation of CPLHTTPFetchEx(). More...

int	CPLHTTPPushFetchCallback (CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
	Installs an alternate callback to the default implementation of CPLHTTPFetchEx(). More...

int	CPLHTTPPopFetchCallback (void)
	Uninstalls a callback set by CPLHTTPPushFetchCallback(). More...

char *	GOA2GetAuthorizationURL (const char *pszScope)
	Return authorization url for a given scope. More...

char *	GOA2GetRefreshToken (const char pszAuthToken, const char pszScope)
	Turn Auth Token into a Refresh Token. More...

char *	GOA2GetAccessToken (const char pszRefreshToken, const char pszScope)
	Fetch access token using refresh token. More...

char **	GOA2GetAccessTokenFromServiceAccount (const char pszPrivateKey, const char pszClientEmail, const char *pszScope, CSLConstList papszAdditionalClaims, CSLConstList papszOptions)
	Fetch access token using Service Account OAuth2. More...

char **	GOA2GetAccessTokenFromCloudEngineVM (CSLConstList papszOptions)
	Fetch access token using Cloud Engine internal REST API. More...

bool	CPLIsMachinePotentiallyGCEInstance ()
	Returns whether the current machine is potentially a Google Compute Engine instance. More...

bool	CPLIsMachineForSureGCEInstance ()
	Returns whether the current machine is surely a Google Compute Engine instance. More...

Detailed Description

Interface for downloading HTTP, FTP documents.

Typedef Documentation

◆ CPLHTTPFetchCallbackFunc

typedef CPLHTTPResult *(* CPLHTTPFetchCallbackFunc) (const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg, void *pUserData)

Callback function to process network requests.

If CLOSE_PERSISTENT is found in papszOptions, no network request should be issued, but a dummy non-null CPLHTTPResult* should be returned by the callback.

Its first arguments are the same as CPLHTTPFetchEx()

Parameters

pszURL	See CPLHTTPFetchEx()
papszOptions	See CPLHTTPFetchEx()
pfnProgress	See CPLHTTPFetchEx()
pProgressArg	See CPLHTTPFetchEx()
pfnWrite	See CPLHTTPFetchEx()
pWriteArg	See CPLHTTPFetchEx()
pUserData	user data value that was passed during CPLHTTPPushFetchCallback()

Returns: nullptr if the request cannot be processed, in which case the previous handler will be used.

Function Documentation

◆ CPLHTTPDestroyMultiResult()

void CPLHTTPDestroyMultiResult	(	CPLHTTPResult **	papsResults,
		int	nCount
	)

Clean the memory associated with the return value of CPLHTTPMultiFetch()

Parameters

papsResults	pointer to the return value of CPLHTTPMultiFetch()
nCount	value of the nURLCount parameter passed to CPLHTTPMultiFetch()

Since: GDAL 2.3

◆ CPLHTTPDestroyResult()

void CPLHTTPDestroyResult ( CPLHTTPResult * psResult )

Clean the memory associated with the return value of CPLHTTPFetch()

Parameters

psResult pointer to the return value of CPLHTTPFetch()

◆ CPLHTTPEnabled()

int CPLHTTPEnabled ( void )

Return if CPLHTTP services can be useful.

Those services depend on GDAL being build with libcurl support.

Returns: TRUE if libcurl support is enabled

◆ CPLHTTPFetch()

CPLHTTPResult * CPLHTTPFetch	(	const char *	pszURL,
		CSLConstList	papszOptions
	)

Fetch a document from an url and return in a string.

Parameters

pszURL valid URL recognized by underlying download library (libcurl)

papszOptions

option list as a NULL-terminated array of strings. May be NULL. The following options are handled :

CONNECTTIMEOUT=val, where val is in seconds (possibly with decimals). This is the maximum delay for the connection to be established before being aborted (GDAL >= 2.2).
TIMEOUT=val, where val is in seconds. This is the maximum delay for the whole request to complete before being aborted.
LOW_SPEED_TIME=val, where val is in seconds. This is the maximum time where the transfer speed should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the transfer to be considered too slow and aborted. (GDAL >= 2.1)
LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME. Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1)
HEADERS=val, where val is an extra header to use when getting a web page. For example "Accept: application/x-ogcwkt"
HEADER_FILE=filename: filename of a text file with "key: value" headers. The content of the file is not cached, and thus it is read again before issuing each HTTP request. (GDAL >= 2.2)
HTTPAUTH=[BASIC/NTLM/NEGOTIATE/ANY/ANYSAFE/BEARER] to specify an authentication scheme to use.
USERPWD=userid:password to specify a user and password for authentication
GSSAPI_DELEGATION=[NONE/POLICY/ALWAYS] set allowed GSS-API delegation. Relevant only with HTTPAUTH=NEGOTIATE (GDAL >= 3.3).
HTTP_BEARER=val set OAuth 2.0 Bearer Access Token. Relevant only with HTTPAUTH=BEARER (GDAL >= 3.9).
POSTFIELDS=val, where val is a nul-terminated string to be passed to the server with a POST request.
PROXY=val, to make requests go through a proxy server, where val is of the form proxy.server.com:port_number. This option affects both HTTP and HTTPS URLs.
HTTPS_PROXY=val (GDAL >= 2.4), the same meaning as PROXY, but this option is taken into account only for HTTPS URLs.
PROXYUSERPWD=val, where val is of the form username:password
PROXYAUTH=[BASIC/NTLM/DIGEST/NEGOTIATE/ANY/ANYSAFE] to specify an proxy authentication scheme to use.
NETRC=[YES/NO] to enable or disable use of $HOME/.netrc (or NETRC_FILE), default YES.
NETRC_FILE=file name to read .netrc info from (GDAL >= 3.7).
CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc.. (GDAL >= 1.9.0)
FORM_FILE_NAME=val, where val is upload file name. If this option and FORM_FILE_PATH present, request type will set to POST.
FORM_FILE_PATH=val, where val is upload file path.
FORM_KEY_0=val...FORM_KEY_N, where val is name of form item.
FORM_VALUE_0=val...FORM_VALUE_N, where val is value of the form item.
FORM_ITEM_COUNT=val, where val is count of form items.
COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2; ...
COOKIEFILE=val, where val is file name to read cookies from (GDAL >= 2.4)
COOKIEJAR=val, where val is file name to store cookies to (GDAL >= 2.4)
MAX_RETRY=val, where val is the maximum number of retry attempts if a 429, 502, 503 or 504 HTTP error occurs. Default is 0. (GDAL >= 2.0)
RETRY_DELAY=val, where val is the number of seconds between retry attempts. Default is 30. (GDAL >= 2.0)
MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)
CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA) bundle file. By default, it will be looked for in a system location. If the CAINFO option is not defined, GDAL will also look in the CURL_CA_BUNDLE and SSL_CERT_FILE environment variables respectively and use the first one found as the CAINFO value (GDAL >= 2.1.3). The GDAL_CURL_CA_BUNDLE environment variable may also be used to set the CAINFO value in GDAL >= 3.2.
HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3). Specify HTTP version to use. Will default to 1.1 generally (except on some controlled environments, like Google Compute Engine VMs, where 2TLS will be the default). Support for HTTP/2 requires curl 7.33 or later, built against nghttp2. "2TLS" means that HTTP/2 will be attempted for HTTPS connections only. Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.
SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines whether the status of the server cert using the "Certificate Status Request" TLS extension (aka. OCSP stapling) should be checked. If this option is enabled but the server does not support the TLS extension, the verification will fail. Default to NO.
USE_CAPI_STORE=YES/NO (GDAL >= 2.3, Windows only): whether CA certificates from the Windows certificate store. Defaults to NO.
TCP_KEEPALIVE=YES/NO (GDAL >= 3.6): whether to enable TCP keep-alive. Defaults to NO
TCP_KEEPIDLE=integer, in seconds (GDAL >= 3.6): keep-alive idle time. Defaults to 60. Only taken into account if TCP_KEEPALIVE=YES.
TCP_KEEPINTVL=integer, in seconds (GDAL >= 3.6): interval time between keep-alive probes. Defaults to 60. Only taken into account if TCP_KEEPALIVE=YES.
USERAGENT=string: value of User-Agent header. Starting with GDAL 3.7, GDAL core sets it by default (during driver initialization) to GDAL/x.y.z where x.y.z is the GDAL version number. Applications may override it with the CPLHTTPSetDefaultUserAgent() function.
SSLCERT=filename (GDAL >= 3.7): Filename of the the SSL client certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLCERT.html
SSLCERTTYPE=string (GDAL >= 3.7): Format of the SSL certificate: "PEM" or "DER". Cf https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html
SSLKEY=filename (GDAL >= 3.7): Private key file for TLS and SSL client certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLKEY.html
KEYPASSWD=string (GDAL >= 3.7): Passphrase to private key. Cf https://curl.se/libcurl/c/CURLOPT_KEYPASSWD.html

Alternatively, if not defined in the papszOptions arguments, the following CONNECTTIMEOUT, TIMEOUT, LOW_SPEED_TIME, LOW_SPEED_LIMIT, USERPWD, PROXY, HTTPS_PROXY, PROXYUSERPWD, PROXYAUTH, NETRC, NETRC_FILE, MAX_RETRY and RETRY_DELAY, HEADER_FILE, HTTP_VERSION, SSL_VERIFYSTATUS, USE_CAPI_STORE, GSSAPI_DELEGATION, HTTP_BEARER, TCP_KEEPALIVE, TCP_KEEPIDLE, TCP_KEEPINTVL, USERAGENT, SSLCERT, SSLCERTTYPE, SSLKEY, KEYPASSWD values are searched in the configuration options respectively named GDAL_HTTP_CONNECTTIMEOUT, GDAL_HTTP_TIMEOUT, GDAL_HTTP_LOW_SPEED_TIME, GDAL_HTTP_LOW_SPEED_LIMIT, GDAL_HTTP_USERPWD, GDAL_HTTP_PROXY, GDAL_HTTPS_PROXY, GDAL_HTTP_PROXYUSERPWD, GDAL_PROXY_AUTH, GDAL_HTTP_NETRC, GDAL_HTTP_NETC_FILE, GDAL_HTTP_MAX_RETRY, GDAL_HTTP_RETRY_DELAY, GDAL_HTTP_HEADER_FILE, GDAL_HTTP_VERSION, GDAL_HTTP_SSL_VERIFYSTATUS, GDAL_HTTP_USE_CAPI_STORE, GDAL_GSSAPI_DELEGATION, GDAL_HTTP_BEARER, GDAL_HTTP_TCP_KEEPALIVE, GDAL_HTTP_TCP_KEEPIDLE, GDAL_HTTP_TCP_KEEPINTVL, GDAL_HTTP_USERAGENT, GDAL_HTTP_SSLCERT, GDAL_HTTP_SSLCERTTYPE, GDAL_HTTP_SSLKEY and GDAL_HTTP_KEYPASSWD.

Starting with GDAL 3.6, the GDAL_HTTP_HEADERS configuration option can also be used to specify a comma separated list of key: value pairs. This is an alternative to the GDAL_HTTP_HEADER_FILE mechanism. If a comma or a double-quote character is needed in the value, then the key: value pair must be enclosed in double-quote characters. In that situation, backslash and double quote character must be backslash-escaped. e.g GDAL_HTTP_HEADERS=Foo: Bar,"Baz: escaped backslash \\, escaped double-quote \", end of value",Another: Header

Starting with GDAL 3.7, the above configuration options can also be specified as path-specific options with VSISetPathSpecificOption().

Returns: a CPLHTTPResult* structure that must be freed by CPLHTTPDestroyResult(), or NULL if libcurl support is disabled

◆ CPLHTTPFetchEx()

CPLHTTPResult * CPLHTTPFetchEx	(	const char *	pszURL,
		CSLConstList	papszOptions,
		GDALProgressFunc	pfnProgress,
		void *	pProgressArg,
		CPLHTTPFetchWriteFunc	pfnWrite,
		void *	pWriteArg
	)

Fetch a document from an url and return in a string.

Parameters

pszURL	Url to fetch document from web.
papszOptions	Option list as a NULL-terminated array of strings. Available keys see in CPLHTTPFetch.
pfnProgress	Callback for reporting algorithm progress matching the GDALProgressFunc() semantics. May be NULL.
pProgressArg	Callback argument passed to pfnProgress.
pfnWrite	Write function pointer matching the CPLHTTPWriteFunc() semantics. May be NULL.
pWriteArg	Argument which will pass to a write function.

Returns: A CPLHTTPResult* structure that must be freed by CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.

◆ CPLHTTPMultiFetch()

CPLHTTPResult ** CPLHTTPMultiFetch	(	const char const	papszURL,
		int	nURLCount,
		int	nMaxSimultaneous,
		CSLConstList	papszOptions
	)

Fetch several documents at once.

Parameters

papszURL	array of valid URLs recognized by underlying download library (libcurl)
nURLCount	number of URLs of papszURL
nMaxSimultaneous	maximum number of downloads to issue simultaneously. Any negative or zer value means unlimited.
papszOptions	option list as a NULL-terminated array of strings. May be NULL. Refer to CPLHTTPFetch() for valid options.

Returns: an array of CPLHTTPResult* structures that must be freed by CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled

Since: GDAL 2.3

◆ CPLHTTPParseMultipartMime()

int CPLHTTPParseMultipartMime ( CPLHTTPResult * psResult )

Parses a MIME multipart message.

This function will iterate over each part and put it in a separate element of the pasMimePart array of the provided psResult structure.

Parameters

psResult pointer to the return value of CPLHTTPFetch()

Returns: TRUE if the message contains MIME multipart message.

◆ CPLHTTPPopFetchCallback()

int CPLHTTPPopFetchCallback ( void )

Uninstalls a callback set by CPLHTTPPushFetchCallback().

See also: CPLHTTPPushFetchCallback()

Returns: TRUE in case of success.

Since: GDAL 3.2

◆ CPLHTTPPushFetchCallback()

int CPLHTTPPushFetchCallback	(	CPLHTTPFetchCallbackFunc	pFunc,
		void *	pUserData
	)

Installs an alternate callback to the default implementation of CPLHTTPFetchEx().

This callback will only be used in the thread where this function has been called. It must be un-installed by CPLHTTPPopFetchCallback(), which must also be called from the same thread.

Parameters

pFunc	Callback function to be called with CPLHTTPFetchEx() is called.
pUserData	Last argument to provide to the pFunc callback.

Returns: TRUE in case of success.

Since: GDAL 3.2

◆ CPLHTTPSetDefaultUserAgent()

void CPLHTTPSetDefaultUserAgent ( const char * pszUserAgent )

Set the default user agent.

GDAL core will by default call this method with "GDAL/x.y.z" where x.y.z is the GDAL version number (during driver initialization). Applications may override it.

Parameters

pszUserAgent String (or nullptr to cancel the default user agent)

Since: GDAL 3.7

◆ CPLHTTPSetFetchCallback()

void CPLHTTPSetFetchCallback	(	CPLHTTPFetchCallbackFunc	pFunc,
		void *	pUserData
	)

Installs an alternate callback to the default implementation of CPLHTTPFetchEx().

This callback will be used by all threads, unless contextual callbacks are installed with CPLHTTPPushFetchCallback().

It is the responsibility of the caller to make sure this function is not called concurrently, or during CPLHTTPFetchEx() execution.

Parameters

pFunc	Callback function to be called with CPLHTTPFetchEx() is called (or NULL to restore default handler)
pUserData	Last argument to provide to the pFunc callback.

Since: GDAL 3.2

◆ CPLIsMachineForSureGCEInstance()

bool CPLIsMachineForSureGCEInstance ( )

Returns whether the current machine is surely a Google Compute Engine instance.

This does a very quick check without network access. Note: only works for Linux GCE instances.

Returns: true if the current machine is surely a GCE instance.

Since: GDAL 2.3

◆ CPLIsMachinePotentiallyGCEInstance()

bool CPLIsMachinePotentiallyGCEInstance ( )

Returns whether the current machine is potentially a Google Compute Engine instance.

This does a very quick check without network access. To confirm if the machine is effectively a GCE instance, metadata.google.internal must be queried.

Returns: true if the current machine is potentially a GCE instance.

Since: GDAL 2.3

◆ GOA2GetAccessToken()

char * GOA2GetAccessToken	(	const char *	pszRefreshToken,
		const char *	pszScope
	)

Fetch access token using refresh token.

The permanent refresh token is used to fetch a temporary (usually one hour) access token using Google OAuth2 web services.

A CPLError will be reported if the request fails for some reason. Common reasons include the refresh token having been revoked by the user or http connection problems.

Parameters

pszRefreshToken	the refresh token from GOA2GetRefreshToken().
pszScope	the scope for which it is valid. Currently unused

Returns: access token, to be freed with CPLFree(), null on failure.

◆ GOA2GetAccessTokenFromCloudEngineVM()

char ** GOA2GetAccessTokenFromCloudEngineVM ( CSLConstList papszOptions )

Fetch access token using Cloud Engine internal REST API.

The default service accounts bound to the current Google Cloud Engine VM is used for OAuth2 authentication

A CPLError will be reported if the request fails for some reason. Common reasons include the refresh token having been revoked by the user or http connection problems.

Parameters

papszOptions NULL terminated list of options. None currently

Returns: a list of key=value pairs, including a access_token and expires_in

Since: GDAL 2.3

◆ GOA2GetAccessTokenFromServiceAccount()

char ** GOA2GetAccessTokenFromServiceAccount	(	const char *	pszPrivateKey,
		const char *	pszClientEmail,
		const char *	pszScope,
		CSLConstList	papszAdditionalClaims,
		CSLConstList	papszOptions
	)

Fetch access token using Service Account OAuth2.

See https://developers.google.com/identity/protocols/OAuth2ServiceAccount

A CPLError will be reported if the request fails for some reason.

Parameters

pszPrivateKey	Private key as a RSA private key
pszClientEmail	Client email
pszScope	the service being requested
papszAdditionalClaims	additional claims, or NULL
papszOptions	NULL terminated list of options. None currently

Returns: a list of key=value pairs, including a access_token and expires_in

Since: GDAL 2.3

See https://developers.google.com/identity/protocols/OAuth2ServiceAccount and https://jwt.io/

◆ GOA2GetAuthorizationURL()

char * GOA2GetAuthorizationURL ( const char * pszScope )

Return authorization url for a given scope.

Returns the URL that a user should visit, and use for authentication in order to get an "auth token" indicating their willingness to use a service.

Note that when the user visits this url they will be asked to login (using a google/gmail/etc) account, and to authorize use of the requested scope for the application "GDAL/OGR". Once they have done so, they will be presented with a lengthy string they should "enter into their application". This is the "auth token" to be passed to GOA2GetRefreshToken(). The "auth token" can only be used once.

This function should never fail.

Parameters

pszScope the service being requested, not yet URL encoded, such as "https://www.googleapis.com/auth/fusiontables".

Returns: the URL to visit - should be freed with CPLFree().

◆ GOA2GetRefreshToken()

char * GOA2GetRefreshToken	(	const char *	pszAuthToken,
		const char *	pszScope
	)

Turn Auth Token into a Refresh Token.

A one time "auth token" provided by the user is turned into a reusable "refresh token" using a google oauth2 web service.

A CPLError will be reported if the translation fails for some reason. Common reasons include the auth token already having been used before, it not being appropriate for the passed scope and configured client api or http connection problems. NULL is returned on error.

Parameters

pszAuthToken	the authorization token from the user.
pszScope	the scope for which it is valid.

Returns: refresh token, to be freed with CPLFree(), null on failure.

Classes

Typedefs

Functions

Detailed Description

Typedef Documentation

◆ CPLHTTPFetchCallbackFunc

Function Documentation

◆ CPLHTTPDestroyMultiResult()

◆ CPLHTTPDestroyResult()

◆ CPLHTTPEnabled()

◆ CPLHTTPFetch()

◆ CPLHTTPFetchEx()

◆ CPLHTTPMultiFetch()

◆ CPLHTTPParseMultipartMime()

◆ CPLHTTPPopFetchCallback()

◆ CPLHTTPPushFetchCallback()

◆ CPLHTTPSetDefaultUserAgent()

◆ CPLHTTPSetFetchCallback()

◆ CPLIsMachineForSureGCEInstance()

◆ CPLIsMachinePotentiallyGCEInstance()

◆ GOA2GetAccessToken()

◆ GOA2GetAccessTokenFromCloudEngineVM()

◆ GOA2GetAccessTokenFromServiceAccount()

◆ GOA2GetAuthorizationURL()

◆ GOA2GetRefreshToken()