GDAL

cpl_aws.h
1/**********************************************************************
2 * $Id$
3 *
4 * Name: cpl_aws.h
5 * Project: CPL - Common Portability Library
6 * Purpose: Amazon Web Services routines
7 * Author: Even Rouault <even.rouault at spatialys.com>
8 *
9 **********************************************************************
10 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11 *
12 * Permission is hereby granted, free of charge, to any person obtaining a
13 * copy of this software and associated documentation files (the "Software"),
14 * to deal in the Software without restriction, including without limitation
15 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 * and/or sell copies of the Software, and to permit persons to whom the
17 * Software is furnished to do so, subject to the following conditions:
18 *
19 * The above copyright notice and this permission notice shall be included
20 * in all copies or substantial portions of the Software.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 * DEALINGS IN THE SOFTWARE.
29 ****************************************************************************/
30
31#ifndef CPL_AWS_INCLUDED_H
32#define CPL_AWS_INCLUDED_H
33
34#ifndef DOXYGEN_SKIP
35
36#ifdef HAVE_CURL
37
38#include <cstddef>
39#include <mutex>
40
41#include "cpl_string.h"
42
43#include <curl/curl.h>
44#include <map>
45
46std::string CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
47std::string CPLGetLowerCaseHexSHA256(const std::string &osStr);
48
49std::string CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50
51std::string CPLAWSURLEncode(const std::string &osURL, bool bEncodeSlash = true);
52
53std::string CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
54 const char *pszKey);
55
56std::string CPLGetAWS_SIGN4_Signature(
57 const std::string &osSecretAccessKey, const std::string &osAccessToken,
58 const std::string &osRegion, const std::string &osRequestPayer,
59 const std::string &osService, const std::string &osVerb,
60 const struct curl_slist *psExistingHeaders, const std::string &osHost,
61 const std::string &osCanonicalURI,
62 const std::string &osCanonicalQueryString,
63 const std::string &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
64 const std::string &osTimestamp, std::string &osSignedHeaders);
65
66std::string CPLGetAWS_SIGN4_Authorization(
67 const std::string &osSecretAccessKey, const std::string &osAccessKeyId,
68 const std::string &osAccessToken, const std::string &osRegion,
69 const std::string &osRequestPayer, const std::string &osService,
70 const std::string &osVerb, const struct curl_slist *psExistingHeaders,
71 const std::string &osHost, const std::string &osCanonicalURI,
72 const std::string &osCanonicalQueryString,
73 const std::string &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
74 const std::string &osTimestamp);
75
76 class IVSIS3LikeHandleHelper
77{
78 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
79
80 protected:
81 std::map<std::string, std::string> m_oMapQueryParameters{};
82
83 virtual void RebuildURL() = 0;
84 std::string GetQueryString(bool bAddEmptyValueAfterEqual) const;
85
86 public:
87 IVSIS3LikeHandleHelper() = default;
88 virtual ~IVSIS3LikeHandleHelper() = default;
89
90 void ResetQueryParameters();
91 void AddQueryParameter(const std::string &osKey,
92 const std::string &osValue);
93
94 virtual struct curl_slist *
95 GetCurlHeaders(const std::string &osVerb,
96 const struct curl_slist *psExistingHeaders,
97 const void *pabyDataContent = nullptr,
98 size_t nBytesContent = 0) const = 0;
99
100 virtual bool AllowAutomaticRedirection()
101 {
102 return true;
103 }
104
105 virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
106 bool /*bSetError*/)
107 {
108 return false;
109 }
110
111 virtual const std::string &GetURL() const = 0;
112 std::string GetURLNoKVP() const;
113
114 virtual std::string GetCopySourceHeader() const
115 {
116 return std::string();
117 }
118
119 virtual const char *GetMetadataDirectiveREPLACE() const
120 {
121 return "";
122 }
123
124 static bool GetBucketAndObjectKey(const char *pszURI,
125 const char *pszFSPrefix,
126 bool bAllowNoObject,
127 std::string &osBucketOut,
128 std::string &osObjectKeyOut);
129
130 static std::string BuildCanonicalizedHeaders(
131 std::map<std::string, std::string> &oSortedMapHeaders,
132 const struct curl_slist *psExistingHeaders,
133 const char *pszHeaderPrefix);
134
135 static std::string GetRFC822DateTime();
136};
137
138 enum class AWSCredentialsSource
139{
140 REGULAR, // credentials from env variables or ~/.aws/crediential
141 EC2, // credentials from EC2 private networking
142 WEB_IDENTITY, // credentials from Web Identity Token
143 // See
144 // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
145 ASSUMED_ROLE // credentials from an STS assumed role
146 // See
147 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
148 // and
149 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
150};
151
152 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
153{
154 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
155
156 std::string m_osURL{};
157 mutable std::string m_osSecretAccessKey{};
158 mutable std::string m_osAccessKeyId{};
159 mutable std::string m_osSessionToken{};
160 std::string m_osEndpoint{};
161 std::string m_osRegion{};
162 std::string m_osRequestPayer{};
163 std::string m_osBucket{};
164 std::string m_osObjectKey{};
165 bool m_bUseHTTPS = false;
166 bool m_bUseVirtualHosting = false;
167 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
168
169 void RebuildURL() override;
170
171 static bool GetOrRefreshTemporaryCredentialsForRole(
172 bool bForceRefresh, std::string &osSecretAccessKey,
173 std::string &osAccessKeyId, std::string &osSessionToken,
174 std::string &osRegion);
175
176 static bool GetConfigurationFromAssumeRoleWithWebIdentity(
177 bool bForceRefresh, const std::string &osPathForOption,
178 const std::string &osRoleArnIn,
179 const std::string &osWebIdentityTokenFileIn,
180 std::string &osSecretAccessKey, std::string &osAccessKeyId,
181 std::string &osSessionToken);
182
183 static bool GetConfigurationFromEC2(bool bForceRefresh,
184 const std::string &osPathForOption,
185 std::string &osSecretAccessKey,
186 std::string &osAccessKeyId,
187 std::string &osSessionToken);
188
189 static bool GetConfigurationFromAWSConfigFiles(
190 const std::string &osPathForOption, const char *pszProfile,
191 std::string &osSecretAccessKey, std::string &osAccessKeyId,
192 std::string &osSessionToken, std::string &osRegion,
193 std::string &osCredentials, std::string &osRoleArn,
194 std::string &osSourceProfile, std::string &osExternalId,
195 std::string &osMFASerial, std::string &osRoleSessionName,
196 std::string &osWebIdentityTokenFile);
197
198 static bool GetConfiguration(const std::string &osPathForOption,
199 CSLConstList papszOptions,
200 std::string &osSecretAccessKey,
201 std::string &osAccessKeyId,
202 std::string &osSessionToken,
203 std::string &osRegion,
204 AWSCredentialsSource &eCredentialsSource);
205
206 void RefreshCredentials(const std::string &osPathForOption,
207 bool bForceRefresh) const;
208
209 protected:
210 public:
211 VSIS3HandleHelper(
212 const std::string &osSecretAccessKey, const std::string &osAccessKeyId,
213 const std::string &osSessionToken, const std::string &osEndpoint,
214 const std::string &osRegion, const std::string &osRequestPayer,
215 const std::string &osBucket, const std::string &osObjectKey,
216 bool bUseHTTPS, bool bUseVirtualHosting,
217 AWSCredentialsSource eCredentialsSource);
218 ~VSIS3HandleHelper();
219
220 static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
221 const char *pszFSPrefix,
222 bool bAllowNoObject,
223 CSLConstList papszOptions = nullptr);
224 static std::string BuildURL(const std::string &osEndpoint,
225 const std::string &osBucket,
226 const std::string &osObjectKey, bool bUseHTTPS,
227 bool bUseVirtualHosting);
228
229 struct curl_slist *
230 GetCurlHeaders(const std::string &osVerb,
231 const struct curl_slist *psExistingHeaders,
232 const void *pabyDataContent = nullptr,
233 size_t nBytesContent = 0) const override;
234
235 bool AllowAutomaticRedirection() override
236 {
237 return false;
238 }
239
240 bool CanRestartOnError(const char *, const char *pszHeaders,
241 bool bSetError) override;
242
243 const std::string &GetURL() const override
244 {
245 return m_osURL;
246 }
247
248 const std::string &GetBucket() const
249 {
250 return m_osBucket;
251 }
252
253 const std::string &GetObjectKey() const
254 {
255 return m_osObjectKey;
256 }
257
258 const std::string &GetEndpoint() const
259 {
260 return m_osEndpoint;
261 }
262
263 const std::string &GetRegion() const
264 {
265 return m_osRegion;
266 }
267
268 const std::string &GetRequestPayer() const
269 {
270 return m_osRequestPayer;
271 }
272
273 bool GetVirtualHosting() const
274 {
275 return m_bUseVirtualHosting;
276 }
277
278 void SetEndpoint(const std::string &osStr);
279 void SetRegion(const std::string &osStr);
280 void SetRequestPayer(const std::string &osStr);
281 void SetVirtualHosting(bool b);
282
283 std::string GetCopySourceHeader() const override
284 {
285 return "x-amz-copy-source";
286 }
287
288 const char *GetMetadataDirectiveREPLACE() const override
289 {
290 return "x-amz-metadata-directive: REPLACE";
291 }
292
293 std::string GetSignedURL(CSLConstList papszOptions);
294
295 static void CleanMutex();
296 static void ClearCache();
297};
298
299 class VSIS3UpdateParams
300{
301 private:
302 std::string m_osRegion{};
303 std::string m_osEndpoint{};
304 std::string m_osRequestPayer{};
305 bool m_bUseVirtualHosting = false;
306
307 explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
308 : m_osRegion(poHelper->GetRegion()),
309 m_osEndpoint(poHelper->GetEndpoint()),
310 m_osRequestPayer(poHelper->GetRequestPayer()),
311 m_bUseVirtualHosting(poHelper->GetVirtualHosting())
312 {
313 }
314
315 void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
316 {
317 poHelper->SetRegion(m_osRegion);
318 poHelper->SetEndpoint(m_osEndpoint);
319 poHelper->SetRequestPayer(m_osRequestPayer);
320 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
321 }
322
323 static std::mutex gsMutex;
324 static std::map<std::string, VSIS3UpdateParams> goMapBucketsToS3Params;
325
326 public:
327 VSIS3UpdateParams() = default;
328
329 static void UpdateMapFromHandle(VSIS3HandleHelper *poS3HandleHelper);
330 static void UpdateHandleFromMap(VSIS3HandleHelper *poS3HandleHelper);
331 static void ClearCache();
332};
333
334#endif /* HAVE_CURL */
335
336#endif /* #ifndef DOXYGEN_SKIP */
337
338#endif /* CPL_AWS_INCLUDED_H */
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:1042
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1183
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition cpl_port.h:215
cpl_string.h
Various convenience functions for working with strings and string lists.