29#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30#define CPL_VSIL_CURL_CLASS_H_INCLUDED
39#include "cpl_vsil_curl_priv.h"
40#include "cpl_mem_cache.h"
42#include "cpl_curl_priv.h"
45#include <condition_variable>
61#define HAVE_CURLINFO_REDIRECT_URL
63void VSICurlStreamingClearCache(
void);
65struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle,
const char *pszURL,
66 const char *
const *papszOptions);
67struct curl_slist *VSICurlMergeHeaders(
struct curl_slist *poDest,
68 struct curl_slist *poSrcToDestroy);
70struct curl_slist *VSICurlSetContentTypeFromExt(
struct curl_slist *polist,
73struct curl_slist *VSICurlSetCreationHeadersFromOptions(
74 struct curl_slist *headers,
CSLConstList papszOptions,
const char *pszPath);
89 unsigned int nGenerationAuthParameters = 0;
90 ExistStatus eExists = EXIST_UNKNOWN;
94 time_t nExpireTimestampLocal = 0;
95 std::string osRedirectURL{};
96 bool bHasComputedFileSize =
false;
97 bool bIsDirectory =
false;
99 bool bS3LikeRedirect =
false;
105 bool bGotFileList =
false;
106 unsigned int nGenerationAuthParameters = 0;
110struct WriteFuncStruct
112 char *pBuffer =
nullptr;
114 bool bIsHTTP =
false;
115 bool bMultiRange =
false;
120 bool bFoundContentRange =
false;
122 bool bInterruptDownload =
false;
123 bool bDetectRangeDownloadingError =
false;
127 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
128 void *pReadCbkUserData =
nullptr;
129 bool bInterrupted =
false;
134 const GByte *pabyData =
nullptr;
136 size_t nTotalSize = 0;
138 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
141 PutData *poThis =
static_cast<PutData *
>(instream);
142 const size_t nSizeMax = size * nitems;
143 const size_t nSizeToWrite =
144 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
145 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
146 poThis->nOff += nSizeToWrite;
157class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
161 struct FilenameOffsetPair
163 std::string filename_;
166 FilenameOffsetPair(
const std::string &filename,
vsi_l_offset offset)
167 : filename_(filename), offset_(offset)
171 bool operator==(
const FilenameOffsetPair &other)
const
173 return filename_ == other.filename_ && offset_ == other.offset_;
177 struct FilenameOffsetPairHasher
179 std::size_t operator()(
const FilenameOffsetPair &k)
const
181 return std::hash<std::string>()(k.filename_) ^
182 std::hash<vsi_l_offset>()(k.offset_);
186 using RegionCacheType = lru11::Cache<
187 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
190 typename std::list<lru11::KeyValuePair<
191 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
192 FilenameOffsetPairHasher>>;
194 std::unique_ptr<RegionCacheType>
195 m_poRegionCacheDoNotUseDirectly{};
197 RegionCacheType *GetRegionCache();
204 lru11::Cache<std::string, bool> oCacheFileProp;
206 int nCachedFilesInDirList = 0;
207 lru11::Cache<std::string, CachedDirList> oCacheDirList;
209 char **ParseHTMLFileList(
const char *pszFilename,
int nMaxFiles,
210 char *pszData,
bool *pbGotFileList);
215 struct RegionInDownload
218 std::condition_variable oCond{};
219 bool bDownloadInProgress =
false;
221 std::string osData{};
224 std::mutex m_oMutex{};
225 std::map<std::string, std::unique_ptr<RegionInDownload>>
226 m_oMapRegionInDownload{};
229 CPLMutex *hMutex =
nullptr;
231 virtual VSICurlHandle *CreateFileHandle(
const char *pszFilename);
232 virtual char **GetFileList(
const char *pszFilename,
int nMaxFiles,
233 bool *pbGotFileList);
235 void RegisterEmptyDir(
const std::string &osDirname);
238 AnalyseS3FileList(
const std::string &osBaseURL,
const char *pszXML,
240 const std::set<std::string> &oSetIgnoredStorageClasses,
243 void AnalyseSwiftFileList(
const std::string &osBaseURL,
244 const std::string &osPrefix,
const char *pszJson,
246 int nMaxFiles,
bool &bIsTruncated,
247 std::string &osNextMarker);
249 static const char *GetOptionsStatic();
251 VSICurlFilesystemHandlerBase();
254 ~VSICurlFilesystemHandlerBase()
override;
256 static bool IsAllowedFilename(
const char *pszFilename);
262 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
263 int nFlags)
override;
264 int Unlink(
const char *pszFilename)
override;
265 int Rename(
const char *oldpath,
const char *newpath)
override;
266 int Mkdir(
const char *pszDirname,
long nMode)
override;
267 int Rmdir(
const char *pszDirname)
override;
268 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles)
override;
269 char **SiblingFiles(
const char *pszFilename)
override;
271 int HasOptimizedReadMultiRange(
const char * )
override
276 const char *GetActualURL(
const char *pszFilename)
override;
278 const char *GetOptions()
override;
280 char **GetFileMetadata(
const char *pszFilename,
const char *pszDomain,
283 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
284 bool *pbGotFileList);
285 void InvalidateDirContent(
const char *pszDirname);
287 virtual const char *GetDebugKey()
const = 0;
289 virtual std::string GetFSPrefix()
const = 0;
290 virtual bool AllowCachedDataFor(
const char *pszFilename);
292 virtual bool IsLocal(
const char * )
override
298 SupportsSequentialWrite(
const char * ,
304 virtual bool SupportsRandomWrite(
const char * ,
310 std::shared_ptr<std::string> GetRegion(
const char *pszURL,
313 void AddRegion(
const char *pszURL,
vsi_l_offset nFileOffsetStart,
314 size_t nSize,
const char *pData);
316 std::pair<bool, std::string>
317 NotifyStartDownloadRegion(
const std::string &osURL,
319 void NotifyStopDownloadRegion(
const std::string &osURL,
321 const std::string &osData);
323 bool GetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
324 void SetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
325 void InvalidateCachedData(
const char *pszURL);
327 CURLM *GetCurlMultiHandleFor(
const std::string &osURL);
329 virtual void ClearCache();
330 virtual void PartialClearCache(
const char *pszFilename);
332 bool GetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
333 void SetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
334 bool ExistsInCacheDirList(
const std::string &osDirname,
bool *pbIsDir);
336 virtual std::string GetURLFromFilename(
const std::string &osFilename);
339 GetStreamingFilename(
const std::string &osFilename)
const override = 0;
341 static std::set<std::string> GetS3IgnoredStorageClasses();
344class VSICurlFilesystemHandler :
public VSICurlFilesystemHandlerBase
349 VSICurlFilesystemHandler() =
default;
351 const char *GetDebugKey()
const override
356 std::string GetFSPrefix()
const override
362 GetStreamingFilename(
const std::string &osFilename)
const override;
374 VSICurlFilesystemHandlerBase *poFS =
nullptr;
376 bool m_bCached =
true;
378 mutable FileProp oFileProp{};
380 mutable std::mutex m_oMutex{};
381 std::string m_osFilename{};
382 char *m_pszURL =
nullptr;
383 mutable std::string m_osQueryString{};
385 char **m_papszHTTPOptions =
nullptr;
388 int nBlocksToDownload = 1;
390 bool bStopOnInterruptUntilUninstall =
false;
391 bool bInterrupted =
false;
392 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
393 void *pReadCbkUserData =
nullptr;
396 double m_dfRetryDelay = 0.0;
400 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
401 const int nBlocks,
const char *pBuffer,
409 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
411 bool m_bUseHead =
false;
412 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
416 mutable bool m_bPlanetaryComputerURLSigning =
false;
417 mutable std::string m_osPlanetaryComputerCollection{};
418 void ManagePlanetaryComputerSigning()
const;
420 int ReadMultiRangeSingleGet(
int nRanges,
void **ppData,
422 const size_t *panSizes);
423 std::string GetRedirectURLIfValid(
bool &bHasExpired)
const;
425 void UpdateRedirectInfo(CURL *hCurlHandle,
426 const WriteFuncStruct &sWriteFuncHeaderData);
429 struct AdviseReadRange
433 std::condition_variable oCV{};
436 std::vector<GByte> abyData{};
439 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
440 std::thread m_oThreadAdviseRead{};
443 virtual struct curl_slist *
444 GetCurlHeaders(
const std::string & ,
445 const struct curl_slist * )
450 virtual bool AllowAutomaticRedirection()
455 virtual bool CanRestartOnError(
const char *,
const char *,
bool)
460 virtual bool UseLimitRangeGetInsteadOfHead()
465 virtual bool IsDirectoryFromExists(
const char * ,
471 virtual void ProcessGetFileSizeResult(
const char * )
475 void SetURL(
const char *pszURL);
477 virtual bool Authenticate(
const char * )
483 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS,
const char *pszFilename,
484 const char *pszURLIn =
nullptr);
485 ~VSICurlHandle()
override;
489 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
490 int ReadMultiRange(
int nRanges,
void **ppData,
492 const size_t *panSizes)
override;
493 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
495 int Flush()
override;
496 int Close()
override;
498 bool HasPRead()
const override
503 size_t PRead(
void *pBuffer,
size_t nSize,
506 void AdviseRead(
int nRanges,
const vsi_l_offset *panOffsets,
507 const size_t *panSizes)
override;
509 size_t GetAdviseReadTotalBytesLimit()
const override;
511 bool IsKnownFileSize()
const
513 return oFileProp.bHasComputedFileSize;
516 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
520 return GetFileSizeOrHeaders(bSetError,
false);
523 bool Exists(
bool bSetError);
525 bool IsDirectory()
const
527 return oFileProp.bIsDirectory;
532 return oFileProp.nMode;
535 time_t GetMTime()
const
537 return oFileProp.mTime;
545 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk,
void *pfnUserData,
546 int bStopOnInterruptUntilUninstall);
547 int UninstallReadCbk();
549 const char *GetURL()
const
559class VSICurlFilesystemHandlerBaseWritable :
public VSICurlFilesystemHandlerBase
564 VSICurlFilesystemHandlerBaseWritable() =
default;
566 virtual VSIVirtualHandleUniquePtr
567 CreateWriteHandle(
const char *pszFilename,
CSLConstList papszOptions) = 0;
573 bool SupportsSequentialWrite(
const char * ,
579 bool SupportsRandomWrite(
const char * ,
587class IVSIS3LikeFSHandler :
public VSICurlFilesystemHandlerBaseWritable
591 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
595 char **GetFileList(
const char *pszFilename,
int nMaxFiles,
596 bool *pbGotFileList)
override;
598 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(
const char *pszURI,
599 bool bAllowNoObject) = 0;
601 virtual int CopyObject(
const char *oldpath,
const char *newpath,
604 int RmdirRecursiveInternal(
const char *pszDirname,
int nBatchSize);
607 IsAllowedHeaderForObjectCreation(
const char * )
612 IVSIS3LikeFSHandler() =
default;
615 int Unlink(
const char *pszFilename)
override;
616 int Mkdir(
const char *pszDirname,
long nMode)
override;
617 int Rmdir(
const char *pszDirname)
override;
618 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
619 int nFlags)
override;
620 int Rename(
const char *oldpath,
const char *newpath)
override;
622 virtual int CopyFile(
const char *pszSource,
const char *pszTarget,
624 const char *
const *papszOptions,
625 GDALProgressFunc pProgressFunc,
626 void *pProgressData)
override;
628 virtual int DeleteObject(
const char *pszFilename);
630 bool Sync(
const char *pszSource,
const char *pszTarget,
631 const char *
const *papszOptions, GDALProgressFunc pProgressFunc,
632 void *pProgressData,
char ***ppapszOutputs)
override;
634 VSIDIR *OpenDir(
const char *pszPath,
int nRecurseDepth,
635 const char *
const *papszOptions)
override;
638 virtual bool SupportsParallelMultipartUpload()
const
643 virtual std::string InitiateMultipartUpload(
644 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
645 int nMaxRetry,
double dfRetryDelay,
CSLConstList papszOptions);
647 UploadPart(
const std::string &osFilename,
int nPartNumber,
649 const void *pabyBuffer,
size_t nBufferSize,
650 IVSIS3LikeHandleHelper *poS3HandleHelper,
int nMaxRetry,
652 virtual bool CompleteMultipart(
const std::string &osFilename,
653 const std::string &osUploadID,
654 const std::vector<std::string> &aosEtags,
656 IVSIS3LikeHandleHelper *poS3HandleHelper,
657 int nMaxRetry,
double dfRetryDelay);
658 virtual bool AbortMultipart(
const std::string &osFilename,
659 const std::string &osUploadID,
660 IVSIS3LikeHandleHelper *poS3HandleHelper,
661 int nMaxRetry,
double dfRetryDelay);
663 bool AbortPendingUploads(
const char *pszFilename)
override;
670class IVSIS3LikeHandle :
public VSICurlHandle
675 bool UseLimitRangeGetInsteadOfHead()
override
680 bool IsDirectoryFromExists(
const char *pszVerb,
int response_code)
override
683 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
684 std::string(m_pszURL).back() ==
'/';
687 void ProcessGetFileSizeResult(
const char *pszContent)
override
689 oFileProp.bIsDirectory =
690 strstr(pszContent,
"ListBucketResult") !=
nullptr;
694 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
695 const char *pszFilename,
const char *pszURLIn)
696 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
700 ~IVSIS3LikeHandle()
override
713 IVSIS3LikeFSHandler *m_poFS =
nullptr;
714 std::string m_osFilename{};
715 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
716 bool m_bUseChunked =
false;
721 int m_nBufferOff = 0;
722 int m_nBufferSize = 0;
723 bool m_bClosed =
false;
724 GByte *m_pabyBuffer =
nullptr;
725 std::string m_osUploadID{};
726 int m_nPartNumber = 0;
727 std::vector<std::string> m_aosEtags{};
728 bool m_bError =
false;
730 CURLM *m_hCurlMulti =
nullptr;
731 CURL *m_hCurl =
nullptr;
732 const void *m_pBuffer =
nullptr;
733 std::string m_osCurlErrBuf{};
734 size_t m_nChunkedBufferOff = 0;
735 size_t m_nChunkedBufferSize = 0;
736 size_t m_nWrittenInPUT = 0;
739 double m_dfRetryDelay = 0.0;
740 WriteFuncStruct m_sWriteFuncHeaderData{};
743 bool DoSinglePartPUT();
745 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
746 size_t nitems,
void *instream);
747 size_t WriteChunked(
const void *pBuffer,
size_t nSize,
size_t nMemb);
748 int FinishChunkedTransfer();
750 void InvalidateParentDirectory();
753 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS,
const char *pszFilename,
754 IVSIS3LikeHandleHelper *poS3HandleHelper,
bool bUseChunked,
756 ~VSIS3WriteHandle()
override;
760 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
761 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
763 int Close()
override;
767 return m_bUseChunked || m_pabyBuffer !=
nullptr;
780 VSICurlFilesystemHandlerBase *m_poFS =
nullptr;
781 std::string m_osFSPrefix{};
782 std::string m_osFilename{};
785 int m_nBufferOff = 0;
786 int m_nBufferSize = 0;
787 int m_nBufferOffReadCallback = 0;
788 bool m_bClosed =
false;
789 GByte *m_pabyBuffer =
nullptr;
790 bool m_bError =
false;
792 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
794 virtual bool Send(
bool bIsLastBlock) = 0;
797 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
798 const char *pszFSPrefix,
const char *pszFilename,
800 virtual ~VSIAppendWriteHandle();
804 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
805 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
807 int Close()
override;
811 return m_pabyBuffer !=
nullptr;
819struct VSIDIRWithMissingDirSynthesis :
public VSIDIR
821 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
824 std::vector<std::string> m_aosSubpathsStack{};
826 void SynthetizeMissingDirectories(
const std::string &osCurSubdir,
827 bool bAddEntryForThisSubdir);
834struct CurlRequestHelper
836 WriteFuncStruct sWriteFuncData{};
837 WriteFuncStruct sWriteFuncHeaderData{};
838 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
841 ~CurlRequestHelper();
842 long perform(CURL *hCurlHandle,
843 struct curl_slist *headers,
844 VSICurlFilesystemHandlerBase *poFS,
845 IVSIS3LikeHandleHelper *poS3HandleHelper);
852class NetworkStatisticsLogger
854 static int gnEnabled;
855 static NetworkStatisticsLogger gInstance;
857 NetworkStatisticsLogger() =
default;
859 std::mutex m_mutex{};
868 GIntBig nGETDownloadedBytes = 0;
870 GIntBig nPOSTDownloadedBytes = 0;
871 GIntBig nPOSTUploadedBytes = 0;
874 enum class ContextPathType
881 struct ContextPathItem
883 ContextPathType eType;
886 ContextPathItem(ContextPathType eTypeIn,
const std::string &osNameIn)
887 : eType(eTypeIn), osName(osNameIn)
891 bool operator<(
const ContextPathItem &other)
const
893 if (
static_cast<int>(eType) <
static_cast<int>(other.eType))
895 if (
static_cast<int>(eType) >
static_cast<int>(other.eType))
897 return osName < other.osName;
904 std::map<ContextPathItem, Stats> children{};
912 std::map<GIntBig, std::vector<ContextPathItem>>
913 m_mapThreadIdToContextPath{};
915 static void ReadEnabled();
917 std::vector<Counters *> GetCountersForContext();
920 static inline bool IsEnabled()
926 return gnEnabled == TRUE;
929 static void EnterFileSystem(
const char *pszName);
931 static void LeaveFileSystem();
933 static void EnterFile(
const char *pszName);
935 static void LeaveFile();
937 static void EnterAction(
const char *pszName);
939 static void LeaveAction();
941 static void LogHEAD();
943 static void LogGET(
size_t nDownloadedBytes);
945 static void LogPUT(
size_t nUploadedBytes);
947 static void LogPOST(
size_t nUploadedBytes,
size_t nDownloadedBytes);
949 static void LogDELETE();
953 static std::string GetReportAsSerializedJSON();
956struct NetworkStatisticsFileSystem
958 inline explicit NetworkStatisticsFileSystem(
const char *pszName)
960 NetworkStatisticsLogger::EnterFileSystem(pszName);
963 inline ~NetworkStatisticsFileSystem()
965 NetworkStatisticsLogger::LeaveFileSystem();
969struct NetworkStatisticsFile
971 inline explicit NetworkStatisticsFile(
const char *pszName)
973 NetworkStatisticsLogger::EnterFile(pszName);
976 inline ~NetworkStatisticsFile()
978 NetworkStatisticsLogger::LeaveFile();
982struct NetworkStatisticsAction
984 inline explicit NetworkStatisticsAction(
const char *pszName)
986 NetworkStatisticsLogger::EnterAction(pszName);
989 inline ~NetworkStatisticsAction()
991 NetworkStatisticsLogger::LeaveAction();
997int VSICURLGetDownloadChunkSize();
999void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct,
VSILFILE *fp,
1000 VSICurlReadCbkFunc pfnReadCbk,
1001 void *pReadCbkUserData);
1002size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
size_t nmemb,
1004void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle =
nullptr);
1005void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
1007int VSICurlParseUnixPermissions(
const char *pszPermissions);
1010bool VSICURLGetCachedFileProp(
const char *pszURL, cpl::FileProp &oFileProp);
1011void VSICURLSetCachedFileProp(
const char *pszURL, cpl::FileProp &oFileProp);
1012void VSICURLInvalidateCachedFileProp(
const char *pszURL);
1013void VSICURLInvalidateCachedFilePropPrefix(
const char *pszURL);
1014void VSICURLDestroyCacheFileProp();
1016void VSICURLMultiCleanup(CURLM *hCurlMultiHandle);
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:57
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:449
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:551
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1183
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:185
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:215
Various convenience functions for working with strings and string lists.
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:402
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
Virtual file handle.
Definition: cpl_vsi_virtual.h:63