GDAL

cpl_aws.h
1/**********************************************************************
2 *
3 * Name: cpl_aws.h
4 * Project: CPL - Common Portability Library
5 * Purpose: Amazon Web Services routines
6 * Author: Even Rouault <even.rouault at spatialys.com>
7 *
8 **********************************************************************
9 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
10 *
11 * SPDX-License-Identifier: MIT
12 ****************************************************************************/
13
14#ifndef CPL_AWS_INCLUDED_H
15#define CPL_AWS_INCLUDED_H
16
17#ifndef DOXYGEN_SKIP
18
19#ifdef HAVE_CURL
20
21#include <cstddef>
22#include <mutex>
23
24#include "cpl_string.h"
25
26#include <curl/curl.h>
27#include <map>
28
29std::string CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
30std::string CPLGetLowerCaseHexSHA256(const std::string &osStr);
31
32std::string CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
33
34std::string CPLAWSURLEncode(const std::string &osURL, bool bEncodeSlash = true);
35
36std::string CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
37 const char *pszKey);
38
39 class IVSIS3LikeHandleHelper
40{
41 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
42
43 protected:
44 std::map<std::string, std::string> m_oMapQueryParameters{};
45
46 virtual void RebuildURL() = 0;
47 std::string GetQueryString(bool bAddEmptyValueAfterEqual) const;
48
49 public:
50 IVSIS3LikeHandleHelper();
51 virtual ~IVSIS3LikeHandleHelper();
52
53 void ResetQueryParameters();
54 void AddQueryParameter(const std::string &osKey,
55 const std::string &osValue);
56
57 virtual struct curl_slist *
58 GetCurlHeaders(const std::string &osVerb, struct curl_slist *psHeaders,
59 const void *pabyDataContent = nullptr,
60 size_t nBytesContent = 0) const = 0;
61
62 virtual bool AllowAutomaticRedirection()
63 {
64 return true;
65 }
66
67 virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
68 bool /*bSetError*/)
69 {
70 return false;
71 }
72
73 virtual const std::string &GetURL() const = 0;
74 std::string GetURLNoKVP() const;
75
76 virtual std::string GetCopySourceHeader() const
77 {
78 return std::string();
79 }
80
81 virtual const char *GetMetadataDirectiveREPLACE() const
82 {
83 return "";
84 }
85
86 static bool GetBucketAndObjectKey(const char *pszURI,
87 const char *pszFSPrefix,
88 bool bAllowNoObject,
89 std::string &osBucketOut,
90 std::string &osObjectKeyOut);
91
92 static std::string BuildCanonicalizedHeaders(
93 std::map<std::string, std::string> &oSortedMapHeaders,
94 const struct curl_slist *psExistingHeaders,
95 const char *pszHeaderPrefix);
96
97 static std::string GetRFC822DateTime();
98};
99
100 enum class AWSCredentialsSource
101{
102 UNINITIALIZED,
103 NO_SIGN_REQUEST,
104 REGULAR, // credentials from env variables or ~/.aws/crediential
105 EC2, // credentials from EC2 private networking
106 WEB_IDENTITY, // credentials from Web Identity Token
107 // See
108 // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
109 ASSUMED_ROLE, // credentials from an STS assumed role
110 // See
111 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
112 // and
113 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
114 SSO, // credentials from Single-Sign On
115 // credentials from credential_process command
116 // See https://docs.aws.amazon.com/sdkref/latest/guide/feature-process-credentials.html
117 CREDENTIAL_PROCESS,
118};
119
120 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
121{
122 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
123
124 std::string m_osURL{};
125 std::string m_osService{};
126 mutable std::string m_osSecretAccessKey{};
127 mutable std::string m_osAccessKeyId{};
128 mutable std::string m_osSessionToken{};
129 std::string m_osS3SessionToken{};
130 std::string m_osEndpoint{};
131 std::string m_osRegion{};
132 std::string m_osRequestPayer{};
133 std::string m_osBucket{};
134 std::string m_osObjectKey{};
135 bool m_bUseHTTPS = false;
136 bool m_bUseVirtualHosting = false;
137 bool m_bIsDirectoryBucket = false;
138 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
139
140 void RebuildURL() override;
141
142 static bool GetOrRefreshTemporaryCredentialsForRole(
143 bool bForceRefresh, std::string &osSecretAccessKey,
144 std::string &osAccessKeyId, std::string &osSessionToken,
145 std::string &osRegion);
146
147 static bool GetOrRefreshTemporaryCredentialsForSSO(
148 bool bForceRefresh, std::string &osSecretAccessKey,
149 std::string &osAccessKeyId, std::string &osSessionToken,
150 std::string &osRegion);
151
152 static bool GetOrRefreshTemporaryCredentialsFromProcess(
153 bool bForceRefresh, std::string &osSecretAccessKey,
154 std::string &osAccessKeyId, std::string &osSessionToken);
155
156 static bool GetConfigurationFromAssumeRoleWithWebIdentity(
157 bool bForceRefresh, const std::string &osPathForOption,
158 const std::string &osRoleArnIn,
159 const std::string &osWebIdentityTokenFileIn,
160 std::string &osSecretAccessKey, std::string &osAccessKeyId,
161 std::string &osSessionToken);
162
163 static bool GetConfigurationFromEC2(bool bForceRefresh,
164 const std::string &osPathForOption,
165 std::string &osSecretAccessKey,
166 std::string &osAccessKeyId,
167 std::string &osSessionToken);
168
169 static bool GetConfigurationFromAWSConfigFiles(
170 const std::string &osPathForOption, const char *pszProfile,
171 std::string &osSecretAccessKey, std::string &osAccessKeyId,
172 std::string &osSessionToken, std::string &osRegion,
173 std::string &osCredentials, std::string &osRoleArn,
174 std::string &osSourceProfile, std::string &osExternalId,
175 std::string &osMFASerial, std::string &osRoleSessionName,
176 std::string &osWebIdentityTokenFile, std::string &osSSOStartURL,
177 std::string &osSSOAccountID, std::string &osSSORoleName,
178 std::string &osSSOSession, std::string &osCredentialProcess);
179
180 static bool GetConfiguration(const std::string &osPathForOption,
181 CSLConstList papszOptions,
182 std::string &osSecretAccessKey,
183 std::string &osAccessKeyId,
184 std::string &osSessionToken,
185 std::string &osRegion,
186 AWSCredentialsSource &eCredentialsSource);
187
188 void RefreshCredentials(const std::string &osPathForOption,
189 bool bForceRefresh) const;
190
191 protected:
192 public:
193 VSIS3HandleHelper(
194 const std::string &osService, const std::string &osSecretAccessKey,
195 const std::string &osAccessKeyId, const std::string &osSessionToken,
196 const std::string &osS3SessionToken, const std::string &osEndpoint,
197 const std::string &osRegion, const std::string &osRequestPayer,
198 const std::string &osBucket, const std::string &osObjectKey,
199 bool bUseHTTPS, bool bUseVirtualHosting,
200 AWSCredentialsSource eCredentialsSource, bool bIsDirectoryBucket);
201 ~VSIS3HandleHelper() override;
202
203 static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
204 const char *pszFSPrefix,
205 bool bAllowNoObject,
206 CSLConstList papszOptions = nullptr);
207 static std::string BuildURL(const std::string &osEndpoint,
208 const std::string &osBucket,
209 const std::string &osObjectKey, bool bUseHTTPS,
210 bool bUseVirtualHosting);
211
212 struct curl_slist *GetCurlHeaders(const std::string &osVerb,
213 struct curl_slist *psHeaders,
214 const void *pabyDataContent = nullptr,
215 size_t nBytesContent = 0) const override;
216
217 bool IsDirectoryBucket() const
218 {
219 return m_bIsDirectoryBucket;
220 }
221
222 bool AllowAutomaticRedirection() override
223 {
224 return false;
225 }
226
227 bool CanRestartOnError(const char *, const char *pszHeaders,
228 bool bSetError) override;
229
230 const std::string &GetURL() const override
231 {
232 return m_osURL;
233 }
234
235 const std::string &GetBucket() const
236 {
237 return m_osBucket;
238 }
239
240 const std::string &GetObjectKey() const
241 {
242 return m_osObjectKey;
243 }
244
245 const std::string &GetEndpoint() const
246 {
247 return m_osEndpoint;
248 }
249
250 const std::string &GetRegion() const
251 {
252 return m_osRegion;
253 }
254
255 const std::string &GetRequestPayer() const
256 {
257 return m_osRequestPayer;
258 }
259
260 bool GetVirtualHosting() const
261 {
262 return m_bUseVirtualHosting;
263 }
264
265 void SetEndpoint(const std::string &osStr);
266 void SetRegion(const std::string &osStr);
267 void SetRequestPayer(const std::string &osStr);
268 void SetVirtualHosting(bool b);
269
270 std::string GetCopySourceHeader() const override
271 {
272 return "x-amz-copy-source";
273 }
274
275 const char *GetMetadataDirectiveREPLACE() const override
276 {
277 return "x-amz-metadata-directive: REPLACE";
278 }
279
280 std::string GetSignedURL(CSLConstList papszOptions);
281
282 static void CleanMutex();
283 static void ClearCache();
284};
285
286 class VSIS3UpdateParams
287{
288 private:
289 std::string m_osRegion{};
290 std::string m_osEndpoint{};
291 std::string m_osRequestPayer{};
292 bool m_bUseVirtualHosting = false;
293
294 explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
295 : m_osRegion(poHelper->GetRegion()),
296 m_osEndpoint(poHelper->GetEndpoint()),
297 m_osRequestPayer(poHelper->GetRequestPayer()),
298 m_bUseVirtualHosting(poHelper->GetVirtualHosting())
299 {
300 }
301
302 void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
303 {
304 poHelper->SetRegion(m_osRegion);
305 poHelper->SetEndpoint(m_osEndpoint);
306 poHelper->SetRequestPayer(m_osRequestPayer);
307 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
308 }
309
310 static std::mutex gsMutex;
311 static std::map<std::string, VSIS3UpdateParams> goMapBucketsToS3Params;
312
313 public:
314 VSIS3UpdateParams() = default;
315
316 static void UpdateMapFromHandle(VSIS3HandleHelper *poS3HandleHelper);
317 static void UpdateHandleFromMap(VSIS3HandleHelper *poS3HandleHelper);
318 static void ClearCache();
319};
320
321#endif /* HAVE_CURL */
322
323#endif /* #ifndef DOXYGEN_SKIP */
324
325#endif /* CPL_AWS_INCLUDED_H */
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:936
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1087
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition cpl_port.h:205
cpl_string.h
Various convenience functions for working with strings and string lists.