GDAL

cpl_http.h
Go to the documentation of this file.
1/******************************************************************************
2 * $Id$
3 *
4 * Project: Common Portability Library
5 * Purpose: Function wrapper for libcurl HTTP access.
6 * Author: Frank Warmerdam, warmerdam@pobox.com
7 *
8 ******************************************************************************
9 * Copyright (c) 2006, Frank Warmerdam
10 * Copyright (c) 2009, Even Rouault <even dot rouault at spatialys.com>
11 *
12 * SPDX-License-Identifier: MIT
13 ****************************************************************************/
14
15#ifndef CPL_HTTP_H_INCLUDED
16#define CPL_HTTP_H_INCLUDED
17
18#include "cpl_conv.h"
19#include "cpl_string.h"
20#include "cpl_progress.h"
21#include "cpl_vsi.h"
22
30#ifndef CPL_HTTP_MAX_RETRY
31#define CPL_HTTP_MAX_RETRY 0
32#endif
33
34#ifndef CPL_HTTP_RETRY_DELAY
35#define CPL_HTTP_RETRY_DELAY 30.0
36#endif
39 CPL_C_START
40
42 typedef struct
43{
char **papszHeaders;
45
GByte *pabyData; int nDataLen;
48} CPLMimePart;
49
51 typedef struct
52{
54 int nStatus;
55
57 char *pszContentType;
58
60 char *pszErrBuf;
61
63 int nDataLen;
65 int nDataAlloc;
66
68 GByte *pabyData;
69
71 char **papszHeaders;
72
74 int nMimePartCount;
75
77 CPLMimePart *pasMimePart;
78
79} CPLHTTPResult;
80
82 typedef size_t (*CPLHTTPFetchWriteFunc)(void *pBuffer, size_t nSize,
83 size_t nMemb, void *pWriteArg);
86 int CPL_DLL CPLHTTPEnabled(void);
87 CPLHTTPResult CPL_DLL *CPLHTTPFetch(const char *pszURL,
88 CSLConstList papszOptions);
89 CPLHTTPResult CPL_DLL *
90 CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
91 GDALProgressFunc pfnProgress, void *pProgressArg,
92 CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg);
93 CPLHTTPResult CPL_DLL **CPLHTTPMultiFetch(const char *const *papszURL,
94 int nURLCount, int nMaxSimultaneous,
95 CSLConstList papszOptions);
96
97 void CPL_DLL CPLHTTPCleanup(void);
98 void CPL_DLL CPLHTTPDestroyResult(CPLHTTPResult *psResult);
99 void CPL_DLL CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount);
100 int CPL_DLL CPLHTTPParseMultipartMime(CPLHTTPResult *psResult);
101
102 void CPL_DLL CPLHTTPSetDefaultUserAgent(const char *pszUserAgent);
103
104/* -------------------------------------------------------------------- */
105/* To install an alternate network layer to the default Curl one */
106/* -------------------------------------------------------------------- */
125 typedef CPLHTTPResult *(*CPLHTTPFetchCallbackFunc)(
126 const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress,
127 void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg,
128 void *pUserData);
129
130 void CPL_DLL CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc,
131 void *pUserData);
132
133 int CPL_DLL CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc,
134 void *pUserData);
135 int CPL_DLL CPLHTTPPopFetchCallback(void);
136
137/* -------------------------------------------------------------------- */
138/* The following is related to OAuth2 authorization around */
139/* google services like fusion tables, and potentially others */
140/* in the future. Code in cpl_google_oauth2.cpp. */
141/* */
142/* These services are built on CPL HTTP services. */
143/* -------------------------------------------------------------------- */
144
145 char CPL_DLL *GOA2GetAuthorizationURL(const char *pszScope);
146 char CPL_DLL *GOA2GetRefreshToken(const char *pszAuthToken,
147 const char *pszScope);
148 char CPL_DLL *GOA2GetAccessToken(const char *pszRefreshToken,
149 const char *pszScope);
150
151 char CPL_DLL **GOA2GetAccessTokenFromServiceAccount(
152 const char *pszPrivateKey, const char *pszClientEmail, const char *pszScope,
153 CSLConstList papszAdditionalClaims, CSLConstList papszOptions);
154
155 char CPL_DLL **GOA2GetAccessTokenFromCloudEngineVM(CSLConstList papszOptions);
156
157 CPL_C_END
158
159#if defined(__cplusplus) && !defined(CPL_SUPRESS_CPLUSPLUS)
161// Not sure if this belong here, used in cpl_http.cpp, cpl_vsil_curl.cpp and
162// frmts/wms/gdalhttp.cpp
163 void CPL_DLL *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
164 const char *const *papszOptions);
165 char **CPLHTTPGetOptionsFromEnv(const char *pszFilename);
166
168 struct CPLHTTPRetryParameters
169{
170 int nMaxRetry = CPL_HTTP_MAX_RETRY;
171 double dfInitialDelay = CPL_HTTP_RETRY_DELAY;
172 std::string osRetryCodes{};
173
174 CPLHTTPRetryParameters() = default;
175 explicit CPLHTTPRetryParameters(const CPLStringList &aosHTTPOptions);
176};
177
179 class CPLHTTPRetryContext
180{
181 public:
182 explicit CPLHTTPRetryContext(const CPLHTTPRetryParameters &oParams);
183
184 bool CanRetry(int response_code, const char *pszErrBuf,
185 const char *pszCurlError);
186 bool CanRetry();
187
189 double GetCurrentDelay() const;
190
192 void ResetCounter()
193 {
194 m_nRetryCount = 0;
195 }
196
197 private:
198 CPLHTTPRetryParameters m_oParameters{};
199 int m_nRetryCount = 0;
200 double m_dfCurDelay = 0.0;
201 double m_dfNextDelay = 0.0;
202};
203
204 void CPL_DLL *CPLHTTPIgnoreSigPipe();
205 void CPL_DLL CPLHTTPRestoreSigPipeHandler(void *old_handler);
206 bool CPLMultiPerformWait(void *hCurlMultiHandle, int &repeats);
209 bool CPL_DLL CPLIsMachinePotentiallyGCEInstance();
210 bool CPLIsMachineForSureGCEInstance();
211
219 class GOA2Manager
220{
221 public:
222 GOA2Manager();
223
225 typedef enum
226 {
227 NONE,
228 GCE,
229 ACCESS_TOKEN_FROM_REFRESH,
230 SERVICE_ACCOUNT
231 } AuthMethod;
232
233 bool SetAuthFromGCE(CSLConstList papszOptions);
234 bool SetAuthFromRefreshToken(const char *pszRefreshToken,
235 const char *pszClientId,
236 const char *pszClientSecret,
237 CSLConstList papszOptions);
238 bool SetAuthFromServiceAccount(const char *pszPrivateKey,
239 const char *pszClientEmail,
240 const char *pszScope,
241 CSLConstList papszAdditionalClaims,
242 CSLConstList papszOptions);
243
245 AuthMethod GetAuthMethod() const
246 {
247 return m_eMethod;
248 }
249
250 const char *GetBearer() const;
251
253 const CPLString &GetPrivateKey() const
254 {
255 return m_osPrivateKey;
256 }
257
259 const CPLString &GetClientEmail() const
260 {
261 return m_osClientEmail;
262 }
263
267 std::string GetKey() const
268 {
269 std::string osKey(std::to_string(static_cast<int>(m_eMethod))
270 .append(",client-id=")
271 .append(m_osClientId)
272 .append(",client-secret=")
273 .append(m_osClientSecret)
274 .append(",refresh-token=")
275 .append(m_osRefreshToken)
276 .append(",private-key=")
277 .append(m_osPrivateKey)
278 .append(",client-email=")
279 .append(m_osClientEmail)
280 .append(",scope=")
281 .append(m_osScope));
282 osKey.append(",additional-claims=");
283 for (const auto *pszOption : m_aosAdditionalClaims)
284 {
285 osKey.append(pszOption);
286 osKey.append("+");
287 }
288 osKey.append(",options=");
289 for (const auto *pszOption : m_aosOptions)
290 {
291 osKey.append(pszOption);
292 osKey.append("+");
293 }
294 return osKey;
295 }
296
297 private:
298 mutable CPLString m_osCurrentBearer{};
299 mutable time_t m_nExpirationTime = 0;
300
301 AuthMethod m_eMethod = NONE;
302
303 // for ACCESS_TOKEN_FROM_REFRESH
304 CPLString m_osClientId{};
305 CPLString m_osClientSecret{};
306 CPLString m_osRefreshToken{};
307
308 // for SERVICE_ACCOUNT
309 CPLString m_osPrivateKey{};
310 CPLString m_osClientEmail{};
311 CPLString m_osScope{};
312 CPLStringList m_aosAdditionalClaims{};
313
314 CPLStringList m_aosOptions{};
315};
316
317#endif // __cplusplus
318
319#endif /* ndef CPL_HTTP_H_INCLUDED */
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition cpl_string.h:436
CPLString
Convenient string class based on std::string.
Definition cpl_string.h:307
GOA2Manager
Manager of Google OAuth2 authentication.
Definition cpl_http.h:220
GOA2Manager::GetClientEmail
const CPLString & GetClientEmail() const
Returns client email for SERVICE_ACCOUNT method.
Definition cpl_http.h:259
GOA2Manager::GOA2Manager
GOA2Manager()
Constructor.
GOA2Manager::GetAuthMethod
AuthMethod GetAuthMethod() const
Returns the authentication method.
Definition cpl_http.h:245
GOA2Manager::SetAuthFromRefreshToken
bool SetAuthFromRefreshToken(const char *pszRefreshToken, const char *pszClientId, const char *pszClientSecret, CSLConstList papszOptions)
Specifies that the authentication will be done using the OAuth2 client id method.
Definition cpl_google_oauth2.cpp:548
GOA2Manager::GetKey
std::string GetKey() const
Returns a key that can be used to uniquely identify the instance parameters (excluding bearer)
Definition cpl_http.h:267
GOA2Manager::SetAuthFromServiceAccount
bool SetAuthFromServiceAccount(const char *pszPrivateKey, const char *pszClientEmail, const char *pszScope, CSLConstList papszAdditionalClaims, CSLConstList papszOptions)
Specifies that the authentication will be done using the OAuth2 service account method.
Definition cpl_google_oauth2.cpp:583
GOA2Manager::GetPrivateKey
const CPLString & GetPrivateKey() const
Returns private key for SERVICE_ACCOUNT method.
Definition cpl_http.h:253
GOA2Manager::GetBearer
const char * GetBearer() const
Return the access token.
Definition cpl_google_oauth2.cpp:626
GOA2Manager::SetAuthFromGCE
bool SetAuthFromGCE(CSLConstList papszOptions)
Specifies that the authentication will be done using the local credentials of the current Google Comp...
Definition cpl_google_oauth2.cpp:524
GOA2Manager::AuthMethod
AuthMethod
Authentication method.
Definition cpl_http.h:226
cpl_conv.h
Various convenience functions for CPL.
CPLHTTPPopFetchCallback
int CPLHTTPPopFetchCallback(void)
Uninstalls a callback set by CPLHTTPPushFetchCallback().
Definition cpl_http.cpp:1004
CPLHTTPMultiFetch
CPLHTTPResult ** CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount, int nMaxSimultaneous, CSLConstList papszOptions)
Fetch several documents at once.
Definition cpl_http.cpp:1765
GOA2GetRefreshToken
char * GOA2GetRefreshToken(const char *pszAuthToken, const char *pszScope)
Turn Auth Token into a Refresh Token.
Definition cpl_google_oauth2.cpp:107
CPLHTTPPushFetchCallback
int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
Installs an alternate callback to the default implementation of CPLHTTPFetchEx().
Definition cpl_http.cpp:984
CPLHTTPCleanup
void CPLHTTPCleanup(void)
Cleanup function to call at application termination.
Definition cpl_http.cpp:2766
GOA2GetAccessTokenFromServiceAccount
char ** GOA2GetAccessTokenFromServiceAccount(const char *pszPrivateKey, const char *pszClientEmail, const char *pszScope, CSLConstList papszAdditionalClaims, CSLConstList papszOptions)
Fetch access token using Service Account OAuth2.
Definition cpl_google_oauth2.cpp:410
CPLIsMachinePotentiallyGCEInstance
bool CPLIsMachinePotentiallyGCEInstance()
Returns whether the current machine is potentially a Google Compute Engine instance.
Definition cpl_google_cloud.cpp:138
CPLHTTPFetch
CPLHTTPResult * CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
Fetch a document from an url and return in a string.
Definition cpl_http.cpp:1239
GOA2GetAccessTokenFromCloudEngineVM
char ** GOA2GetAccessTokenFromCloudEngineVM(CSLConstList papszOptions)
Fetch access token using Cloud Engine internal REST API.
Definition cpl_google_oauth2.cpp:370
CPLIsMachineForSureGCEInstance
bool CPLIsMachineForSureGCEInstance()
Returns whether the current machine is surely a Google Compute Engine instance.
Definition cpl_google_cloud.cpp:91
CPLHTTPSetFetchCallback
void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
Installs an alternate callback to the default implementation of CPLHTTPFetchEx().
Definition cpl_http.cpp:961
CPLHTTPFetchCallbackFunc
CPLHTTPResult *(* CPLHTTPFetchCallbackFunc)(const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg, void *pUserData)
Callback function to process network requests.
Definition cpl_http.h:125
GOA2GetAuthorizationURL
char * GOA2GetAuthorizationURL(const char *pszScope)
Return authorization url for a given scope.
Definition cpl_google_oauth2.cpp:72
CPLHTTPFetchEx
CPLHTTPResult * CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void *pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
Fetch a document from an url and return in a string.
Definition cpl_http.cpp:1259
GOA2GetAccessToken
char * GOA2GetAccessToken(const char *pszRefreshToken, const char *pszScope)
Fetch access token using refresh token.
Definition cpl_google_oauth2.cpp:339
CPLHTTPEnabled
int CPLHTTPEnabled(void)
Return if CPLHTTP services can be useful.
Definition cpl_http.cpp:2749
CPLHTTPDestroyResult
void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
Clean the memory associated with the return value of CPLHTTPFetch()
Definition cpl_http.cpp:2822
CPLHTTPDestroyMultiResult
void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
Clean the memory associated with the return value of CPLHTTPMultiFetch()
Definition cpl_http.cpp:2020
CPLHTTPSetDefaultUserAgent
void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
Set the default user agent.
Definition cpl_http.cpp:486
CPLHTTPParseMultipartMime
int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
Parses a MIME multipart message.
Definition cpl_http.cpp:2855
CPL_C_END
#define CPL_C_END
Macro to end a block of C symbols.
Definition cpl_port.h:283
CPL_C_START
#define CPL_C_START
Macro to start a block of C symbols.
Definition cpl_port.h:279
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1179
GByte
unsigned char GByte
Unsigned byte type.
Definition cpl_port.h:169
cpl_string.h
Various convenience functions for working with strings and string lists.
cpl_vsi.h
Standard C Covers.
CPLHTTPResult
Definition cpl_http.h:52
CPLHTTPResult::pabyData
GByte * pabyData
Definition cpl_http.h:68
CPLHTTPResult::nStatus
int nStatus
Definition cpl_http.h:54
CPLHTTPResult::pasMimePart
CPLMimePart * pasMimePart
Definition cpl_http.h:77
CPLHTTPResult::pszContentType
char * pszContentType
Definition cpl_http.h:57
CPLHTTPResult::nDataAlloc
int nDataAlloc
Definition cpl_http.h:65
CPLHTTPResult::papszHeaders
char ** papszHeaders
Definition cpl_http.h:71
CPLHTTPResult::pszErrBuf
char * pszErrBuf
Definition cpl_http.h:60
CPLHTTPResult::nMimePartCount
int nMimePartCount
Definition cpl_http.h:74
CPLHTTPResult::nDataLen
int nDataLen
Definition cpl_http.h:63
CPLMimePart
Definition cpl_http.h:43