GDAL

cpl_aws.h
1/**********************************************************************
2 * $Id$
3 *
4 * Name: cpl_aws.h
5 * Project: CPL - Common Portability Library
6 * Purpose: Amazon Web Services routines
7 * Author: Even Rouault <even.rouault at spatialys.com>
8 *
9 **********************************************************************
10 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11 *
12 * SPDX-License-Identifier: MIT
13 ****************************************************************************/
14
15#ifndef CPL_AWS_INCLUDED_H
16#define CPL_AWS_INCLUDED_H
17
18#ifndef DOXYGEN_SKIP
19
20#ifdef HAVE_CURL
21
22#include <cstddef>
23#include <mutex>
24
25#include "cpl_string.h"
26
27#include <curl/curl.h>
28#include <map>
29
30std::string CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
31std::string CPLGetLowerCaseHexSHA256(const std::string &osStr);
32
33std::string CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
34
35std::string CPLAWSURLEncode(const std::string &osURL, bool bEncodeSlash = true);
36
37std::string CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
38 const char *pszKey);
39
40std::string CPLGetAWS_SIGN4_Signature(
41 const std::string &osSecretAccessKey, const std::string &osAccessToken,
42 const std::string &osRegion, const std::string &osRequestPayer,
43 const std::string &osService, const std::string &osVerb,
44 const struct curl_slist *psExistingHeaders, const std::string &osHost,
45 const std::string &osCanonicalURI,
46 const std::string &osCanonicalQueryString,
47 const std::string &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
48 const std::string &osTimestamp, std::string &osSignedHeaders);
49
50std::string CPLGetAWS_SIGN4_Authorization(
51 const std::string &osSecretAccessKey, const std::string &osAccessKeyId,
52 const std::string &osAccessToken, const std::string &osRegion,
53 const std::string &osRequestPayer, const std::string &osService,
54 const std::string &osVerb, const struct curl_slist *psExistingHeaders,
55 const std::string &osHost, const std::string &osCanonicalURI,
56 const std::string &osCanonicalQueryString,
57 const std::string &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
58 const std::string &osTimestamp);
59
60 class IVSIS3LikeHandleHelper
61{
62 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
63
64 protected:
65 std::map<std::string, std::string> m_oMapQueryParameters{};
66
67 virtual void RebuildURL() = 0;
68 std::string GetQueryString(bool bAddEmptyValueAfterEqual) const;
69
70 public:
71 IVSIS3LikeHandleHelper() = default;
72 virtual ~IVSIS3LikeHandleHelper() = default;
73
74 void ResetQueryParameters();
75 void AddQueryParameter(const std::string &osKey,
76 const std::string &osValue);
77
78 virtual struct curl_slist *
79 GetCurlHeaders(const std::string &osVerb,
80 const struct curl_slist *psExistingHeaders,
81 const void *pabyDataContent = nullptr,
82 size_t nBytesContent = 0) const = 0;
83
84 virtual bool AllowAutomaticRedirection()
85 {
86 return true;
87 }
88
89 virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
90 bool /*bSetError*/)
91 {
92 return false;
93 }
94
95 virtual const std::string &GetURL() const = 0;
96 std::string GetURLNoKVP() const;
97
98 virtual std::string GetCopySourceHeader() const
99 {
100 return std::string();
101 }
102
103 virtual const char *GetMetadataDirectiveREPLACE() const
104 {
105 return "";
106 }
107
108 static bool GetBucketAndObjectKey(const char *pszURI,
109 const char *pszFSPrefix,
110 bool bAllowNoObject,
111 std::string &osBucketOut,
112 std::string &osObjectKeyOut);
113
114 static std::string BuildCanonicalizedHeaders(
115 std::map<std::string, std::string> &oSortedMapHeaders,
116 const struct curl_slist *psExistingHeaders,
117 const char *pszHeaderPrefix);
118
119 static std::string GetRFC822DateTime();
120};
121
122 enum class AWSCredentialsSource
123{
124 REGULAR, // credentials from env variables or ~/.aws/crediential
125 EC2, // credentials from EC2 private networking
126 WEB_IDENTITY, // credentials from Web Identity Token
127 // See
128 // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
129 ASSUMED_ROLE, // credentials from an STS assumed role
130 // See
131 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
132 // and
133 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
134 SSO, // credentials from Single-Sign On
135};
136
137 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
138{
139 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
140
141 std::string m_osURL{};
142 mutable std::string m_osSecretAccessKey{};
143 mutable std::string m_osAccessKeyId{};
144 mutable std::string m_osSessionToken{};
145 std::string m_osEndpoint{};
146 std::string m_osRegion{};
147 std::string m_osRequestPayer{};
148 std::string m_osBucket{};
149 std::string m_osObjectKey{};
150 bool m_bUseHTTPS = false;
151 bool m_bUseVirtualHosting = false;
152 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
153
154 void RebuildURL() override;
155
156 static bool GetOrRefreshTemporaryCredentialsForRole(
157 bool bForceRefresh, std::string &osSecretAccessKey,
158 std::string &osAccessKeyId, std::string &osSessionToken,
159 std::string &osRegion);
160
161 static bool GetOrRefreshTemporaryCredentialsForSSO(
162 bool bForceRefresh, std::string &osSecretAccessKey,
163 std::string &osAccessKeyId, std::string &osSessionToken,
164 std::string &osRegion);
165
166 static bool GetConfigurationFromAssumeRoleWithWebIdentity(
167 bool bForceRefresh, const std::string &osPathForOption,
168 const std::string &osRoleArnIn,
169 const std::string &osWebIdentityTokenFileIn,
170 std::string &osSecretAccessKey, std::string &osAccessKeyId,
171 std::string &osSessionToken);
172
173 static bool GetConfigurationFromEC2(bool bForceRefresh,
174 const std::string &osPathForOption,
175 std::string &osSecretAccessKey,
176 std::string &osAccessKeyId,
177 std::string &osSessionToken);
178
179 static bool GetConfigurationFromAWSConfigFiles(
180 const std::string &osPathForOption, const char *pszProfile,
181 std::string &osSecretAccessKey, std::string &osAccessKeyId,
182 std::string &osSessionToken, std::string &osRegion,
183 std::string &osCredentials, std::string &osRoleArn,
184 std::string &osSourceProfile, std::string &osExternalId,
185 std::string &osMFASerial, std::string &osRoleSessionName,
186 std::string &osWebIdentityTokenFile, std::string &osSSOStartURL,
187 std::string &osSSOAccountID, std::string &osSSORoleName);
188
189 static bool GetConfiguration(const std::string &osPathForOption,
190 CSLConstList papszOptions,
191 std::string &osSecretAccessKey,
192 std::string &osAccessKeyId,
193 std::string &osSessionToken,
194 std::string &osRegion,
195 AWSCredentialsSource &eCredentialsSource);
196
197 void RefreshCredentials(const std::string &osPathForOption,
198 bool bForceRefresh) const;
199
200 protected:
201 public:
202 VSIS3HandleHelper(
203 const std::string &osSecretAccessKey, const std::string &osAccessKeyId,
204 const std::string &osSessionToken, const std::string &osEndpoint,
205 const std::string &osRegion, const std::string &osRequestPayer,
206 const std::string &osBucket, const std::string &osObjectKey,
207 bool bUseHTTPS, bool bUseVirtualHosting,
208 AWSCredentialsSource eCredentialsSource);
209 ~VSIS3HandleHelper();
210
211 static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
212 const char *pszFSPrefix,
213 bool bAllowNoObject,
214 CSLConstList papszOptions = nullptr);
215 static std::string BuildURL(const std::string &osEndpoint,
216 const std::string &osBucket,
217 const std::string &osObjectKey, bool bUseHTTPS,
218 bool bUseVirtualHosting);
219
220 struct curl_slist *
221 GetCurlHeaders(const std::string &osVerb,
222 const struct curl_slist *psExistingHeaders,
223 const void *pabyDataContent = nullptr,
224 size_t nBytesContent = 0) const override;
225
226 bool AllowAutomaticRedirection() override
227 {
228 return false;
229 }
230
231 bool CanRestartOnError(const char *, const char *pszHeaders,
232 bool bSetError) override;
233
234 const std::string &GetURL() const override
235 {
236 return m_osURL;
237 }
238
239 const std::string &GetBucket() const
240 {
241 return m_osBucket;
242 }
243
244 const std::string &GetObjectKey() const
245 {
246 return m_osObjectKey;
247 }
248
249 const std::string &GetEndpoint() const
250 {
251 return m_osEndpoint;
252 }
253
254 const std::string &GetRegion() const
255 {
256 return m_osRegion;
257 }
258
259 const std::string &GetRequestPayer() const
260 {
261 return m_osRequestPayer;
262 }
263
264 bool GetVirtualHosting() const
265 {
266 return m_bUseVirtualHosting;
267 }
268
269 void SetEndpoint(const std::string &osStr);
270 void SetRegion(const std::string &osStr);
271 void SetRequestPayer(const std::string &osStr);
272 void SetVirtualHosting(bool b);
273
274 std::string GetCopySourceHeader() const override
275 {
276 return "x-amz-copy-source";
277 }
278
279 const char *GetMetadataDirectiveREPLACE() const override
280 {
281 return "x-amz-metadata-directive: REPLACE";
282 }
283
284 std::string GetSignedURL(CSLConstList papszOptions);
285
286 static void CleanMutex();
287 static void ClearCache();
288};
289
290 class VSIS3UpdateParams
291{
292 private:
293 std::string m_osRegion{};
294 std::string m_osEndpoint{};
295 std::string m_osRequestPayer{};
296 bool m_bUseVirtualHosting = false;
297
298 explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
299 : m_osRegion(poHelper->GetRegion()),
300 m_osEndpoint(poHelper->GetEndpoint()),
301 m_osRequestPayer(poHelper->GetRequestPayer()),
302 m_bUseVirtualHosting(poHelper->GetVirtualHosting())
303 {
304 }
305
306 void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
307 {
308 poHelper->SetRegion(m_osRegion);
309 poHelper->SetEndpoint(m_osEndpoint);
310 poHelper->SetRequestPayer(m_osRequestPayer);
311 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
312 }
313
314 static std::mutex gsMutex;
315 static std::map<std::string, VSIS3UpdateParams> goMapBucketsToS3Params;
316
317 public:
318 VSIS3UpdateParams() = default;
319
320 static void UpdateMapFromHandle(VSIS3HandleHelper *poS3HandleHelper);
321 static void UpdateHandleFromMap(VSIS3HandleHelper *poS3HandleHelper);
322 static void ClearCache();
323};
324
325#endif /* HAVE_CURL */
326
327#endif /* #ifndef DOXYGEN_SKIP */
328
329#endif /* CPL_AWS_INCLUDED_H */
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:1030
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1179
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition cpl_port.h:199
cpl_string.h
Various convenience functions for working with strings and string lists.