GDAL

cpl_vsil_curl_class.h
1/******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
14#define CPL_VSIL_CURL_CLASS_H_INCLUDED
15
16#ifdef HAVE_CURL
17
18#include "cpl_aws.h"
19#include "cpl_azure.h"
20#include "cpl_port.h"
21#include "cpl_json.h"
22#include "cpl_string.h"
23#include "cpl_vsil_curl_priv.h"
24#include "cpl_mem_cache.h"
25
26#include "cpl_curl_priv.h"
27
28#include <algorithm>
29#include <atomic>
30#include <condition_variable>
31#include <set>
32#include <map>
33#include <memory>
34#include <mutex>
35#include <thread>
36#include <utility>
37
38// To avoid aliasing to CopyFile to CopyFileA on Windows
39#ifdef CopyFile
40#undef CopyFile
41#endif
42
44
45// Leave it for backward compatibility, but deprecate.
46#define HAVE_CURLINFO_REDIRECT_URL
47
48 void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
49
50 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
51 const char *const *papszOptions);
52 struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
53 struct curl_slist *poSrcToDestroy);
54
55 struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
56 const char *pszPath);
57
58 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
59 struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
60
61 namespace cpl
62{
63
64 typedef enum
65{
66 EXIST_UNKNOWN = -1,
67 EXIST_NO,
68 EXIST_YES,
69} ExistStatus;
70
71 class FileProp
72{
73 public:
74 unsigned int nGenerationAuthParameters = 0;
75 ExistStatus eExists = EXIST_UNKNOWN;
76 int nHTTPCode = 0;
77 vsi_l_offset fileSize = 0;
78 time_t mTime = 0;
79 time_t nExpireTimestampLocal = 0;
80 std::string osRedirectURL{};
81 bool bHasComputedFileSize = false;
82 bool bIsDirectory = false;
83 bool bIsAzureFolder = false;
84 int nMode = 0; // st_mode member of struct stat
85 bool bS3LikeRedirect = false;
86 std::string ETag{};
87};
88
89 struct CachedDirList
90{
91 bool bGotFileList = false;
92 unsigned int nGenerationAuthParameters = 0;
93 CPLStringList oFileList{}; /* only file name without path */
94};
95
96 struct WriteFuncStruct
97{
98 char *pBuffer = nullptr;
99 size_t nSize = 0;
100 bool bIsHTTP = false;
101 bool bMultiRange = false;
102 vsi_l_offset nStartOffset = 0;
103 vsi_l_offset nEndOffset = 0;
104 int nHTTPCode = 0; // potentially after redirect
105 int nFirstHTTPCode = 0; // the one of the redirect
106 vsi_l_offset nContentLength = 0;
107 bool bFoundContentRange = false;
108 bool bError = false;
109 bool bInterruptDownload = false;
110 bool bDetectRangeDownloadingError = false;
111 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
112
113 VSILFILE *fp = nullptr;
114 VSICurlReadCbkFunc pfnReadCbk = nullptr;
115 void *pReadCbkUserData = nullptr;
116 bool bInterrupted = false;
117};
118
119 struct PutData
120{
121 const GByte *pabyData = nullptr;
122 size_t nOff = 0;
123 size_t nTotalSize = 0;
124
125 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
126 void *instream)
127 {
128 PutData *poThis = static_cast<PutData *>(instream);
129 const size_t nSizeMax = size * nitems;
130 const size_t nSizeToWrite =
131 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
132 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
133 poThis->nOff += nSizeToWrite;
134 return nSizeToWrite;
135 }
136};
137
138/************************************************************************/
139/* VSICurlFilesystemHandler */
140/************************************************************************/
141
142 class VSICurlHandle;
143
144 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
145{
146 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
147
148 struct FilenameOffsetPair
149 {
150 std::string filename_;
151 vsi_l_offset offset_;
152
153 FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
154 : filename_(filename), offset_(offset)
155 {
156 }
157
158 bool operator==(const FilenameOffsetPair &other) const
159 {
160 return filename_ == other.filename_ && offset_ == other.offset_;
161 }
162 };
163
164 struct FilenameOffsetPairHasher
165 {
166 std::size_t operator()(const FilenameOffsetPair &k) const
167 {
168 return std::hash<std::string>()(k.filename_) ^
169 std::hash<vsi_l_offset>()(k.offset_);
170 }
171 };
172
173 using RegionCacheType = lru11::Cache<
174 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
175 std::unordered_map<
176 FilenameOffsetPair,
177 typename std::list<lru11::KeyValuePair<
178 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
179 FilenameOffsetPairHasher>>;
180
181 std::unique_ptr<RegionCacheType>
182 m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
183 // GetRegionCache();
184 RegionCacheType *GetRegionCache();
185
186 // LRU cache that just keeps in memory if this file system handler is
187 // spposed to know the file properties of a file. The actual cache is a
188 // shared one among all network file systems.
189 // The aim of that design is that invalidating /vsis3/foo results in
190 // /vsis3_streaming/foo to be invalidated as well.
191 lru11::Cache<std::string, bool> oCacheFileProp;
192
193 int nCachedFilesInDirList = 0;
194 lru11::Cache<std::string, CachedDirList> oCacheDirList;
195
196 char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
197 char *pszData, bool *pbGotFileList);
198
199 // Data structure and map to store regions that are in progress, to
200 // avoid simultaneous downloads of the same region in different threads
201 // Cf https://github.com/OSGeo/gdal/issues/8041
202 struct RegionInDownload
203 {
204 std::mutex oMutex{};
205 std::condition_variable oCond{};
206 bool bDownloadInProgress = false;
207 int nWaiters = 0;
208 std::string osData{};
209 };
210
211 std::mutex m_oMutex{};
212 std::map<std::string, std::unique_ptr<RegionInDownload>>
213 m_oMapRegionInDownload{};
214
215 protected:
216 CPLMutex *hMutex = nullptr;
217
218 virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
219 virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
220 bool *pbGotFileList);
221
222 void RegisterEmptyDir(const std::string &osDirname);
223
224 bool
225 AnalyseS3FileList(const std::string &osBaseURL, const char *pszXML,
226 CPLStringList &osFileList, int nMaxFiles,
227 const std::set<std::string> &oSetIgnoredStorageClasses,
228 bool &bIsTruncated);
229
230 void AnalyseSwiftFileList(const std::string &osBaseURL,
231 const std::string &osPrefix, const char *pszJson,
232 CPLStringList &osFileList, int nMaxFilesThisQuery,
233 int nMaxFiles, bool &bIsTruncated,
234 std::string &osNextMarker);
235
236 static const char *GetOptionsStatic();
237
238 VSICurlFilesystemHandlerBase();
239
240 public:
241 ~VSICurlFilesystemHandlerBase() override;
242
243 static bool IsAllowedFilename(const char *pszFilename);
244
245 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
246 bool bSetError,
247 CSLConstList /* papszOptions */) override;
248
249 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
250 int nFlags) override;
251 int Unlink(const char *pszFilename) override;
252 int Rename(const char *oldpath, const char *newpath) override;
253 int Mkdir(const char *pszDirname, long nMode) override;
254 int Rmdir(const char *pszDirname) override;
255 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
256 char **SiblingFiles(const char *pszFilename) override;
257
258 int HasOptimizedReadMultiRange(const char * /* pszPath */) override
259 {
260 return true;
261 }
262
263 const char *GetActualURL(const char *pszFilename) override;
264
265 const char *GetOptions() override;
266
267 char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
268 CSLConstList papszOptions) override;
269
270 char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
271 bool *pbGotFileList);
272 void InvalidateDirContent(const char *pszDirname);
273
274 virtual const char *GetDebugKey() const = 0;
275
276 virtual std::string GetFSPrefix() const = 0;
277 virtual bool AllowCachedDataFor(const char *pszFilename);
278
279 virtual bool IsLocal(const char * /* pszPath */) override
280 {
281 return false;
282 }
283
284 virtual bool
285 SupportsSequentialWrite(const char * /* pszPath */,
286 bool /* bAllowLocalTempFile */) override
287 {
288 return false;
289 }
290
291 virtual bool SupportsRandomWrite(const char * /* pszPath */,
292 bool /* bAllowLocalTempFile */) override
293 {
294 return false;
295 }
296
297 std::shared_ptr<std::string> GetRegion(const char *pszURL,
298 vsi_l_offset nFileOffsetStart);
299
300 void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
301 size_t nSize, const char *pData);
302
303 std::pair<bool, std::string>
304 NotifyStartDownloadRegion(const std::string &osURL,
305 vsi_l_offset startOffset, int nBlocks);
306 void NotifyStopDownloadRegion(const std::string &osURL,
307 vsi_l_offset startOffset, int nBlocks,
308 const std::string &osData);
309
310 bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
311 void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
312 void InvalidateCachedData(const char *pszURL);
313
314 CURLM *GetCurlMultiHandleFor(const std::string &osURL);
315
316 virtual void ClearCache();
317 virtual void PartialClearCache(const char *pszFilename);
318
319 bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
320 void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
321 bool ExistsInCacheDirList(const std::string &osDirname, bool *pbIsDir);
322
323 virtual std::string GetURLFromFilename(const std::string &osFilename) const;
324
325 std::string
326 GetStreamingFilename(const std::string &osFilename) const override = 0;
327
328 static std::set<std::string> GetS3IgnoredStorageClasses();
329};
330
331 class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
332{
333 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
334
335 public:
336 VSICurlFilesystemHandler() = default;
337
338 const char *GetDebugKey() const override
339 {
340 return "VSICURL";
341 }
342
343 std::string GetFSPrefix() const override
344 {
345 return "/vsicurl/";
346 }
347
348 std::string
349 GetStreamingFilename(const std::string &osFilename) const override;
350};
351
352/************************************************************************/
353/* VSICurlHandle */
354/************************************************************************/
355
356 class VSICurlHandle : public VSIVirtualHandle
357{
358 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
359
360 protected:
361 VSICurlFilesystemHandlerBase *poFS = nullptr;
362
363 bool m_bCached = true;
364
365 mutable FileProp oFileProp{};
366
367 mutable std::mutex m_oMutex{};
368 std::string m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
369 char *m_pszURL = nullptr; // e.g "http://example.com/foo"
370 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
371
372 CPLStringList m_aosHTTPOptions{};
373 CPLHTTPRetryParameters
374 m_oRetryParameters; // must be initialized in constructor
375
376 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
377 int nBlocksToDownload = 1;
378
379 bool bStopOnInterruptUntilUninstall = false;
380 bool bInterrupted = false;
381 VSICurlReadCbkFunc pfnReadCbk = nullptr;
382 void *pReadCbkUserData = nullptr;
383
384 CPLStringList m_aosHeaders{};
385
386 void DownloadRegionPostProcess(const vsi_l_offset startOffset,
387 const int nBlocks, const char *pBuffer,
388 size_t nSize);
389
390 private:
391 vsi_l_offset curOffset = 0;
392
393 bool bEOF = false;
394 bool bError = false;
395
396 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
397
398 bool m_bUseHead = false;
399 bool m_bUseRedirectURLIfNoQueryStringParams = false;
400
401 mutable std::atomic<bool> m_bInterrupt = false;
402
403 // Specific to Planetary Computer signing:
404 // https://planetarycomputer.microsoft.com/docs/concepts/sas/
405 mutable bool m_bPlanetaryComputerURLSigning = false;
406 mutable std::string m_osPlanetaryComputerCollection{};
407 void ManagePlanetaryComputerSigning() const;
408
409 int ReadMultiRangeSingleGet(int nRanges, void **ppData,
410 const vsi_l_offset *panOffsets,
411 const size_t *panSizes);
412 std::string GetRedirectURLIfValid(bool &bHasExpired,
413 CPLStringList &aosHTTPOptions) const;
414
415 void UpdateRedirectInfo(CURL *hCurlHandle,
416 const WriteFuncStruct &sWriteFuncHeaderData);
417
418 // Used by AdviseRead()
419 struct AdviseReadRange
420 {
421 bool bDone = false;
422 std::mutex oMutex{};
423 std::condition_variable oCV{};
424 vsi_l_offset nStartOffset = 0;
425 size_t nSize = 0;
426 std::vector<GByte> abyData{};
427 };
428
429 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
430 std::thread m_oThreadAdviseRead{};
431
432 protected:
433 virtual struct curl_slist *
434 GetCurlHeaders(const std::string & /*osVerb*/,
435 const struct curl_slist * /* psExistingHeaders */)
436 {
437 return nullptr;
438 }
439
440 virtual bool AllowAutomaticRedirection()
441 {
442 return true;
443 }
444
445 virtual bool CanRestartOnError(const char *, const char *, bool)
446 {
447 return false;
448 }
449
450 virtual bool UseLimitRangeGetInsteadOfHead()
451 {
452 return false;
453 }
454
455 virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
456 int /*response_code*/)
457 {
458 return false;
459 }
460
461 virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
462 {
463 }
464
465 void SetURL(const char *pszURL);
466
467 virtual bool Authenticate(const char * /* pszFilename */)
468 {
469 return false;
470 }
471
472 public:
473 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
474 const char *pszURLIn = nullptr);
475 ~VSICurlHandle() override;
476
477 int Seek(vsi_l_offset nOffset, int nWhence) override;
478 vsi_l_offset Tell() override;
479 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
480 int ReadMultiRange(int nRanges, void **ppData,
481 const vsi_l_offset *panOffsets,
482 const size_t *panSizes) override;
483 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
484 void ClearErr() override;
485 int Eof() override;
486 int Error() override;
487 int Flush() override;
488 int Close() override;
489
490 void Interrupt() override
491 {
492 m_bInterrupt = true;
493 }
494
495 bool HasPRead() const override
496 {
497 return true;
498 }
499
500 size_t PRead(void *pBuffer, size_t nSize,
501 vsi_l_offset nOffset) const override;
502
503 void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
504 const size_t *panSizes) override;
505
506 size_t GetAdviseReadTotalBytesLimit() const override;
507
508 bool IsKnownFileSize() const
509 {
510 return oFileProp.bHasComputedFileSize;
511 }
512
513 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
514
515 virtual vsi_l_offset GetFileSize(bool bSetError)
516 {
517 return GetFileSizeOrHeaders(bSetError, false);
518 }
519
520 bool Exists(bool bSetError);
521
522 bool IsDirectory() const
523 {
524 return oFileProp.bIsDirectory;
525 }
526
527 int GetMode() const
528 {
529 return oFileProp.nMode;
530 }
531
532 time_t GetMTime() const
533 {
534 return oFileProp.mTime;
535 }
536
537 const CPLStringList &GetHeaders()
538 {
539 return m_aosHeaders;
540 }
541
542 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
543 int bStopOnInterruptUntilUninstall);
544 int UninstallReadCbk();
545
546 const char *GetURL() const
547 {
548 return m_pszURL;
549 }
550};
551
552/************************************************************************/
553/* VSICurlFilesystemHandlerBaseWritable */
554/************************************************************************/
555
556 class VSICurlFilesystemHandlerBaseWritable : public VSICurlFilesystemHandlerBase
557{
558 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBaseWritable)
559
560 protected:
561 VSICurlFilesystemHandlerBaseWritable() = default;
562
563 virtual VSIVirtualHandleUniquePtr
564 CreateWriteHandle(const char *pszFilename, CSLConstList papszOptions) = 0;
565
566 public:
567 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
568 bool bSetError, CSLConstList papszOptions) override;
569
570 bool SupportsSequentialWrite(const char * /* pszPath */,
571 bool /* bAllowLocalTempFile */) override
572 {
573 return true;
574 }
575
576 bool SupportsRandomWrite(const char * /* pszPath */,
577 bool /* bAllowLocalTempFile */) override;
578};
579
580/************************************************************************/
581/* IVSIS3LikeFSHandler */
582/************************************************************************/
583
584 class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBaseWritable
585{
586 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
587
588 virtual int MkdirInternal(const char *pszDirname, long nMode,
589 bool bDoStatCheck);
590
591 protected:
592 char **GetFileList(const char *pszFilename, int nMaxFiles,
593 bool *pbGotFileList) override;
594
595 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
596 bool bAllowNoObject) = 0;
597
598 virtual int CopyObject(const char *oldpath, const char *newpath,
599 CSLConstList papszMetadata);
600
601 int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
602
603 virtual bool
604 IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
605 {
606 return false;
607 }
608
609 IVSIS3LikeFSHandler() = default;
610
611 public:
612 int Unlink(const char *pszFilename) override;
613 int Mkdir(const char *pszDirname, long nMode) override;
614 int Rmdir(const char *pszDirname) override;
615 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
616 int nFlags) override;
617 int Rename(const char *oldpath, const char *newpath) override;
618
619 virtual int CopyFile(const char *pszSource, const char *pszTarget,
620 VSILFILE *fpSource, vsi_l_offset nSourceSize,
621 const char *const *papszOptions,
622 GDALProgressFunc pProgressFunc,
623 void *pProgressData) override;
624
625 virtual int DeleteObject(const char *pszFilename);
626
627 virtual int *DeleteObjectBatch(CSLConstList papszFilesOrDirs);
628
629 bool Sync(const char *pszSource, const char *pszTarget,
630 const char *const *papszOptions, GDALProgressFunc pProgressFunc,
631 void *pProgressData, char ***ppapszOutputs) override;
632
633 VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
634 const char *const *papszOptions) override;
635};
636
637/************************************************************************/
638/* IVSIS3LikeFSHandlerWithMultipartUpload */
639/************************************************************************/
640
641 class IVSIS3LikeFSHandlerWithMultipartUpload : public IVSIS3LikeFSHandler
642{
643 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandlerWithMultipartUpload)
644
645 protected:
646 IVSIS3LikeFSHandlerWithMultipartUpload() = default;
647
648 public:
649 virtual bool SupportsNonSequentialMultipartUpload() const
650 {
651 return true;
652 }
653
654 virtual bool SupportsParallelMultipartUpload() const
655 {
656 return true;
657 }
658
659 virtual bool SupportsMultipartAbort() const = 0;
660
661 size_t GetUploadChunkSizeInBytes(const char *pszFilename,
662 const char *pszSpecifiedValInBytes);
663
664 virtual int CopyFileRestartable(const char *pszSource,
665 const char *pszTarget,
666 const char *pszInputPayload,
667 char **ppszOutputPayload,
668 CSLConstList papszOptions,
669 GDALProgressFunc pProgressFunc,
670 void *pProgressData) override;
671
673 // Limit currently used by S3 and GS.
674 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
675 // and https://cloud.google.com/storage/quotas#requests
676 virtual int GetMaximumPartCount()
677 {
678 return 10000;
679 }
680
682 // Limit currently used by S3 and GS.
683 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
684 // and https://cloud.google.com/storage/quotas#requests
685 virtual int GetMinimumPartSizeInMiB()
686 {
687 return 5;
688 }
689
691 // Limit currently used by S3 and GS.
692 // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
693 // and https://cloud.google.com/storage/quotas#requests
694 virtual int GetMaximumPartSizeInMiB()
695 {
696#if SIZEOF_VOIDP == 8
697 return 5 * 1024;
698#else
699 // Cannot be larger than 4, otherwise integer overflow would occur
700 // 1 GiB is the maximum reasonable value on a 32-bit machine
701 return 1 * 1024;
702#endif
703 }
704
706 virtual int GetDefaultPartSizeInMiB()
707 {
708 return 50;
709 }
710
711 virtual std::string
712 InitiateMultipartUpload(const std::string &osFilename,
713 IVSIS3LikeHandleHelper *poS3HandleHelper,
714 const CPLHTTPRetryParameters &oRetryParameters,
715 CSLConstList papszOptions);
716
717 virtual std::string
718 UploadPart(const std::string &osFilename, int nPartNumber,
719 const std::string &osUploadID, vsi_l_offset nPosition,
720 const void *pabyBuffer, size_t nBufferSize,
721 IVSIS3LikeHandleHelper *poS3HandleHelper,
722 const CPLHTTPRetryParameters &oRetryParameters,
723 CSLConstList papszOptions);
724
725 virtual bool CompleteMultipart(
726 const std::string &osFilename, const std::string &osUploadID,
727 const std::vector<std::string> &aosEtags, vsi_l_offset nTotalSize,
728 IVSIS3LikeHandleHelper *poS3HandleHelper,
729 const CPLHTTPRetryParameters &oRetryParameters);
730
731 virtual bool AbortMultipart(const std::string &osFilename,
732 const std::string &osUploadID,
733 IVSIS3LikeHandleHelper *poS3HandleHelper,
734 const CPLHTTPRetryParameters &oRetryParameters);
735
736 bool AbortPendingUploads(const char *pszFilename) override;
737
738 bool MultipartUploadGetCapabilities(int *pbNonSequentialUploadSupported,
739 int *pbParallelUploadSupported,
740 int *pbAbortSupported,
741 size_t *pnMinPartSize,
742 size_t *pnMaxPartSize,
743 int *pnMaxPartCount) override;
744
745 char *MultipartUploadStart(const char *pszFilename,
746 CSLConstList papszOptions) override;
747
748 char *MultipartUploadAddPart(const char *pszFilename,
749 const char *pszUploadId, int nPartNumber,
750 vsi_l_offset nFileOffset, const void *pData,
751 size_t nDataLength,
752 CSLConstList papszOptions) override;
753
754 bool MultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
755 size_t nPartIdsCount,
756 const char *const *apszPartIds,
757 vsi_l_offset nTotalSize,
758 CSLConstList papszOptions) override;
759
760 bool MultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
761 CSLConstList papszOptions) override;
762};
763
764/************************************************************************/
765/* IVSIS3LikeHandle */
766/************************************************************************/
767
768 class IVSIS3LikeHandle : public VSICurlHandle
769{
770 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
771
772 protected:
773 bool UseLimitRangeGetInsteadOfHead() override
774 {
775 return true;
776 }
777
778 bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
779 {
780 // A bit dirty, but on S3, a GET on a existing directory returns a 416
781 return response_code == 416 && EQUAL(pszVerb, "GET") &&
782 std::string(m_pszURL).back() == '/';
783 }
784
785 void ProcessGetFileSizeResult(const char *pszContent) override
786 {
787 oFileProp.bIsDirectory =
788 strstr(pszContent, "ListBucketResult") != nullptr;
789 }
790
791 public:
792 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
793 const char *pszFilename, const char *pszURLIn)
794 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
795 {
796 }
797
798 ~IVSIS3LikeHandle() override
799 {
800 }
801};
802
803/************************************************************************/
804/* VSIMultipartWriteHandle */
805/************************************************************************/
806
807 class VSIMultipartWriteHandle final : public VSIVirtualHandle
808{
809 CPL_DISALLOW_COPY_ASSIGN(VSIMultipartWriteHandle)
810
811 IVSIS3LikeFSHandlerWithMultipartUpload *m_poFS = nullptr;
812 std::string m_osFilename{};
813 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
814 CPLStringList m_aosOptions{};
815 CPLStringList m_aosHTTPOptions{};
816 CPLHTTPRetryParameters m_oRetryParameters;
817
818 vsi_l_offset m_nCurOffset = 0;
819 size_t m_nBufferOff = 0;
820 size_t m_nBufferSize = 0;
821 bool m_bClosed = false;
822 GByte *m_pabyBuffer = nullptr;
823 std::string m_osUploadID{};
824 int m_nPartNumber = 0;
825 std::vector<std::string> m_aosEtags{};
826 bool m_bError = false;
827
828 WriteFuncStruct m_sWriteFuncHeaderData{};
829
830 bool UploadPart();
831 bool DoSinglePartPUT();
832
833 void InvalidateParentDirectory();
834
835 public:
836 VSIMultipartWriteHandle(IVSIS3LikeFSHandlerWithMultipartUpload *poFS,
837 const char *pszFilename,
838 IVSIS3LikeHandleHelper *poS3HandleHelper,
839 CSLConstList papszOptions);
840 ~VSIMultipartWriteHandle() override;
841
842 int Seek(vsi_l_offset nOffset, int nWhence) override;
843 vsi_l_offset Tell() override;
844 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
845 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
846
847 void ClearErr() override
848 {
849 }
850
851 int Error() override
852 {
853 return FALSE;
854 }
855
856 int Eof() override
857 {
858 return FALSE;
859 }
860
861 int Close() override;
862
863 bool IsOK()
864 {
865 return m_pabyBuffer != nullptr;
866 }
867};
868
869/************************************************************************/
870/* VSIChunkedWriteHandle() */
871/************************************************************************/
872
876 class VSIChunkedWriteHandle final : public VSIVirtualHandle
877{
878 CPL_DISALLOW_COPY_ASSIGN(VSIChunkedWriteHandle)
879
880 IVSIS3LikeFSHandler *m_poFS = nullptr;
881 std::string m_osFilename{};
882 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
883 CPLStringList m_aosOptions{};
884 CPLStringList m_aosHTTPOptions{};
885 CPLHTTPRetryParameters m_oRetryParameters;
886
887 vsi_l_offset m_nCurOffset = 0;
888 size_t m_nBufferOff = 0;
889 bool m_bError = false;
890 bool m_bClosed = false;
891
892 CURLM *m_hCurlMulti = nullptr;
893 CURL *m_hCurl = nullptr;
894 const void *m_pBuffer = nullptr;
895 std::string m_osCurlErrBuf{};
896 size_t m_nChunkedBufferOff = 0;
897 size_t m_nChunkedBufferSize = 0;
898 size_t m_nWrittenInPUT = 0;
899
900 WriteFuncStruct m_sWriteFuncHeaderData{};
901
902 static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
903 size_t nitems, void *instream);
904 int FinishChunkedTransfer();
905
906 bool DoEmptyPUT();
907
908 void InvalidateParentDirectory();
909
910 public:
911 VSIChunkedWriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
912 IVSIS3LikeHandleHelper *poS3HandleHelper,
913 CSLConstList papszOptions);
914 virtual ~VSIChunkedWriteHandle();
915
916 int Seek(vsi_l_offset nOffset, int nWhence) override;
917 vsi_l_offset Tell() override;
918 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
919 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
920
921 void ClearErr() override
922 {
923 }
924
925 int Error() override
926 {
927 return FALSE;
928 }
929
930 int Eof() override
931 {
932 return FALSE;
933 }
934
935 int Close() override;
936};
937
938/************************************************************************/
939/* VSIAppendWriteHandle */
940/************************************************************************/
941
942 class VSIAppendWriteHandle CPL_NON_FINAL : public VSIVirtualHandle
943{
944 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
945
946 protected:
947 VSICurlFilesystemHandlerBase *m_poFS = nullptr;
948 std::string m_osFSPrefix{};
949 std::string m_osFilename{};
950 CPLHTTPRetryParameters m_oRetryParameters{};
951
952 vsi_l_offset m_nCurOffset = 0;
953 int m_nBufferOff = 0;
954 int m_nBufferSize = 0;
955 int m_nBufferOffReadCallback = 0;
956 bool m_bClosed = false;
957 GByte *m_pabyBuffer = nullptr;
958 bool m_bError = false;
959
960 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
961 void *instream);
962 virtual bool Send(bool bIsLastBlock) = 0;
963
964 public:
965 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
966 const char *pszFSPrefix, const char *pszFilename,
967 int nChunkSize);
968 virtual ~VSIAppendWriteHandle();
969
970 int Seek(vsi_l_offset nOffset, int nWhence) override;
971 vsi_l_offset Tell() override;
972 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
973 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
974
975 void ClearErr() override
976 {
977 }
978
979 int Error() override
980 {
981 return FALSE;
982 }
983
984 int Eof() override
985 {
986 return FALSE;
987 }
988
989 int Close() override;
990
991 bool IsOK()
992 {
993 return m_pabyBuffer != nullptr;
994 }
995};
996
997/************************************************************************/
998/* VSIDIRWithMissingDirSynthesis */
999/************************************************************************/
1000
1001 struct VSIDIRWithMissingDirSynthesis : public VSIDIR
1002{
1003 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
1004
1005 protected:
1006 std::vector<std::string> m_aosSubpathsStack{};
1007
1008 void SynthetizeMissingDirectories(const std::string &osCurSubdir,
1009 bool bAddEntryForThisSubdir);
1010};
1011
1012/************************************************************************/
1013/* CurlRequestHelper */
1014/************************************************************************/
1015
1016 struct CurlRequestHelper
1017{
1018 WriteFuncStruct sWriteFuncData{};
1019 WriteFuncStruct sWriteFuncHeaderData{};
1020 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1021
1022 CurlRequestHelper();
1023 ~CurlRequestHelper();
1024 long perform(CURL *hCurlHandle,
1025 struct curl_slist *headers, // ownership transferred
1026 VSICurlFilesystemHandlerBase *poFS,
1027 IVSIS3LikeHandleHelper *poS3HandleHelper);
1028};
1029
1030/************************************************************************/
1031/* NetworkStatisticsLogger */
1032/************************************************************************/
1033
1034 class NetworkStatisticsLogger
1035{
1036 static int gnEnabled;
1037 static NetworkStatisticsLogger gInstance;
1038
1039 NetworkStatisticsLogger() = default;
1040
1041 std::mutex m_mutex{};
1042
1043 struct Counters
1044 {
1045 GIntBig nHEAD = 0;
1046 GIntBig nGET = 0;
1047 GIntBig nPUT = 0;
1048 GIntBig nPOST = 0;
1049 GIntBig nDELETE = 0;
1050 GIntBig nGETDownloadedBytes = 0;
1051 GIntBig nPUTUploadedBytes = 0;
1052 GIntBig nPOSTDownloadedBytes = 0;
1053 GIntBig nPOSTUploadedBytes = 0;
1054 };
1055
1056 enum class ContextPathType
1057 {
1058 FILESYSTEM,
1059 FILE,
1060 ACTION,
1061 };
1062
1063 struct ContextPathItem
1064 {
1065 ContextPathType eType;
1066 std::string osName;
1067
1068 ContextPathItem(ContextPathType eTypeIn, const std::string &osNameIn)
1069 : eType(eTypeIn), osName(osNameIn)
1070 {
1071 }
1072
1073 bool operator<(const ContextPathItem &other) const
1074 {
1075 if (static_cast<int>(eType) < static_cast<int>(other.eType))
1076 return true;
1077 if (static_cast<int>(eType) > static_cast<int>(other.eType))
1078 return false;
1079 return osName < other.osName;
1080 }
1081 };
1082
1083 struct Stats
1084 {
1085 Counters counters{};
1086 std::map<ContextPathItem, Stats> children{};
1087
1088 void AsJSON(CPLJSONObject &oJSON) const;
1089 };
1090
1091 // Workaround bug in Coverity Scan
1092 // coverity[generated_default_constructor_used_in_field_initializer]
1093 Stats m_stats{};
1094 std::map<GIntBig, std::vector<ContextPathItem>>
1095 m_mapThreadIdToContextPath{};
1096
1097 static void ReadEnabled();
1098
1099 std::vector<Counters *> GetCountersForContext();
1100
1101 public:
1102 static inline bool IsEnabled()
1103 {
1104 if (gnEnabled < 0)
1105 {
1106 ReadEnabled();
1107 }
1108 return gnEnabled == TRUE;
1109 }
1110
1111 static void EnterFileSystem(const char *pszName);
1112
1113 static void LeaveFileSystem();
1114
1115 static void EnterFile(const char *pszName);
1116
1117 static void LeaveFile();
1118
1119 static void EnterAction(const char *pszName);
1120
1121 static void LeaveAction();
1122
1123 static void LogHEAD();
1124
1125 static void LogGET(size_t nDownloadedBytes);
1126
1127 static void LogPUT(size_t nUploadedBytes);
1128
1129 static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
1130
1131 static void LogDELETE();
1132
1133 static void Reset();
1134
1135 static std::string GetReportAsSerializedJSON();
1136};
1137
1138 struct NetworkStatisticsFileSystem
1139{
1140 inline explicit NetworkStatisticsFileSystem(const char *pszName)
1141 {
1142 NetworkStatisticsLogger::EnterFileSystem(pszName);
1143 }
1144
1145 inline ~NetworkStatisticsFileSystem()
1146 {
1147 NetworkStatisticsLogger::LeaveFileSystem();
1148 }
1149};
1150
1151 struct NetworkStatisticsFile
1152{
1153 inline explicit NetworkStatisticsFile(const char *pszName)
1154 {
1155 NetworkStatisticsLogger::EnterFile(pszName);
1156 }
1157
1158 inline ~NetworkStatisticsFile()
1159 {
1160 NetworkStatisticsLogger::LeaveFile();
1161 }
1162};
1163
1164 struct NetworkStatisticsAction
1165{
1166 inline explicit NetworkStatisticsAction(const char *pszName)
1167 {
1168 NetworkStatisticsLogger::EnterAction(pszName);
1169 }
1170
1171 inline ~NetworkStatisticsAction()
1172 {
1173 NetworkStatisticsLogger::LeaveAction();
1174 }
1175};
1176
1177} // namespace cpl
1178
1179 int VSICURLGetDownloadChunkSize();
1180
1181 void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp,
1182 VSICurlReadCbkFunc pfnReadCbk,
1183 void *pReadCbkUserData);
1184 size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
1185 void *req);
1186 void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr,
1187 std::atomic<bool> *pbInterrupt = nullptr);
1188 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
1189
1190 int VSICurlParseUnixPermissions(const char *pszPermissions);
1191
1192// Cache of file properties (size, etc.)
1193 bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1194 void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1195 void VSICURLInvalidateCachedFileProp(const char *pszURL);
1196 void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
1197 void VSICURLDestroyCacheFileProp();
1198
1199 void VSICURLMultiCleanup(CURLM *hCurlMultiHandle);
1200
1202
1203#endif // HAVE_CURL
1204
1205#endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition cpl_json.h:41
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition cpl_string.h:436
cpl_json.h
Interface for read and write JSON documents.
cpl_port.h
Core portability definitions for CPL.
CPL_NON_FINAL
#define CPL_NON_FINAL
Mark that a class is explicitly recognized as non-final.
Definition cpl_port.h:1023
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition cpl_port.h:535
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:1030
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1179
GByte
unsigned char GByte
Unsigned byte type.
Definition cpl_port.h:169
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition cpl_port.h:199
cpl_string.h
Various convenience functions for working with strings and string lists.
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition cpl_vsi.h:189
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition cpl_vsi.h:132
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition cpl_vsi.h:388
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition cpl_vsi.h:130
VSIVirtualHandle
Virtual file handle.
Definition cpl_vsi_virtual.h:47