GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_port.h"
36 #include "cpl_string.h"
37 #include "cpl_vsil_curl_priv.h"
38 #include "cpl_mem_cache.h"
39 
40 #include <curl/curl.h>
41 
42 #include <set>
43 #include <map>
44 #include <memory>
45 
47 
48 // 7.18.1
49 #if LIBCURL_VERSION_NUM >= 0x071201
50 #define HAVE_CURLINFO_REDIRECT_URL
51 #endif
52 
53 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
54 
55 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
56  const char * const* papszOptions);
57 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
58  struct curl_slist* poSrcToDestroy );
59 
60 namespace cpl {
61 
62 typedef enum
63 {
64  EXIST_UNKNOWN = -1,
65  EXIST_NO,
66  EXIST_YES,
67 } ExistStatus;
68 
69 class FileProp
70 {
71  public:
72  ExistStatus eExists = EXIST_UNKNOWN;
73  vsi_l_offset fileSize = 0;
74  time_t mTime = 0;
75  time_t nExpireTimestampLocal = 0;
76  CPLString osRedirectURL{};
77  bool bHasComputedFileSize = false;
78  bool bIsDirectory = false;
79  bool bS3LikeRedirect = false;
80  CPLString ETag{};
81 };
82 
83 typedef struct
84 {
85  bool bGotFileList = false;
86  CPLStringList oFileList{}; /* only file name without path */
87 } CachedDirList;
88 
89 typedef struct
90 {
91  char* pBuffer;
92  size_t nSize;
93  bool bIsHTTP;
94  bool bIsInHeader;
95  bool bMultiRange;
96  vsi_l_offset nStartOffset;
97  vsi_l_offset nEndOffset;
98  int nHTTPCode;
99  vsi_l_offset nContentLength;
100  bool bFoundContentRange;
101  bool bError;
102  bool bDownloadHeaderOnly;
103  bool bDetectRangeDownloadingError;
104  GIntBig nTimestampDate; // Corresponds to Date: header field
105 
106  VSILFILE *fp;
107  VSICurlReadCbkFunc pfnReadCbk;
108  void *pReadCbkUserData;
109  bool bInterrupted;
110 
111 #if LIBCURL_VERSION_NUM < 0x073600
112  // Workaround to ignore extra HTTP response headers from
113  // proxies in older versions of curl.
114  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
115  bool bIsProxyConnectHeader;
116 #endif
117 } WriteFuncStruct;
118 
119 /************************************************************************/
120 /* VSICurlFilesystemHandler */
121 /************************************************************************/
122 
123 class VSICurlHandle;
124 
125 class VSICurlFilesystemHandler : public VSIFilesystemHandler
126 {
127  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
128 
129  struct FilenameOffsetPair
130  {
131  std::string filename_;
132  vsi_l_offset offset_;
133 
134  FilenameOffsetPair(const std::string& filename,
135  vsi_l_offset offset) :
136  filename_(filename), offset_(offset) {}
137 
138  bool operator==(const FilenameOffsetPair& other) const
139  {
140  return filename_ == other.filename_ &&
141  offset_ == other.offset_;
142  }
143  };
144  struct FilenameOffsetPairHasher
145  {
146  std::size_t operator()(const FilenameOffsetPair& k) const
147  {
148  return std::hash<std::string>()(k.filename_) ^
149  std::hash<vsi_l_offset>()(k.offset_);
150  }
151  };
152 
153  using RegionCacheType =
154  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
155  lru11::NullLock,
156  std::unordered_map<
157  FilenameOffsetPair,
158  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
159  std::shared_ptr<std::string>>>::iterator,
160  FilenameOffsetPairHasher>>;
161 
162  RegionCacheType oRegionCache;
163 
164  lru11::Cache<std::string, FileProp> oCacheFileProp;
165 
166  int nCachedFilesInDirList = 0;
167  lru11::Cache<std::string, CachedDirList> oCacheDirList;
168 
169  char** ParseHTMLFileList(const char* pszFilename,
170  int nMaxFiles,
171  char* pszData,
172  bool* pbGotFileList);
173 
174 protected:
175  CPLMutex *hMutex = nullptr;
176 
177  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
178  virtual char** GetFileList(const char *pszFilename,
179  int nMaxFiles,
180  bool* pbGotFileList);
181 
182  void RegisterEmptyDir( const CPLString& osDirname );
183 
184  bool AnalyseS3FileList( const CPLString& osBaseURL,
185  const char* pszXML,
186  CPLStringList& osFileList,
187  int nMaxFiles,
188  bool bIgnoreGlacierStorageClass,
189  bool& bIsTruncated );
190 
191  void AnalyseSwiftFileList( const CPLString& osBaseURL,
192  const CPLString& osPrefix,
193  const char* pszJson,
194  CPLStringList& osFileList,
195  int nMaxFilesThisQuery,
196  int nMaxFiles,
197  bool& bIsTruncated,
198  CPLString& osNextMarker );
199 
200  static const char* GetOptionsStatic();
201 
202  static bool IsAllowedFilename( const char* pszFilename );
203 
204 public:
205  VSICurlFilesystemHandler();
206  ~VSICurlFilesystemHandler() override;
207 
208  VSIVirtualHandle *Open( const char *pszFilename,
209  const char *pszAccess,
210  bool bSetError ) override;
211 
212  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
213  int nFlags ) override;
214  int Unlink( const char *pszFilename ) override;
215  int Rename( const char *oldpath, const char *newpath ) override;
216  int Mkdir( const char *pszDirname, long nMode ) override;
217  int Rmdir( const char *pszDirname ) override;
218  char **ReadDir( const char *pszDirname ) override
219  { return ReadDirEx(pszDirname, 0); }
220  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
221 
222  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
223  override { return true; }
224 
225  const char* GetActualURL(const char* pszFilename) override;
226 
227  const char* GetOptions() override;
228 
229  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
230  bool* pbGotFileList );
231  void InvalidateDirContent( const char *pszDirname );
232 
233  virtual CPLString GetFSPrefix() { return "/vsicurl/"; }
234  virtual bool AllowCachedDataFor(const char* pszFilename);
235 
236  std::shared_ptr<std::string> GetRegion( const char* pszURL,
237  vsi_l_offset nFileOffsetStart );
238 
239  void AddRegion( const char* pszURL,
240  vsi_l_offset nFileOffsetStart,
241  size_t nSize,
242  const char *pData );
243 
244  bool GetCachedFileProp( const char* pszURL,
245  FileProp& oFileProp );
246  void SetCachedFileProp( const char* pszURL,
247  const FileProp& oFileProp );
248  void InvalidateCachedData( const char* pszURL );
249 
250  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
251 
252  virtual void ClearCache();
253  virtual void PartialClearCache(const char* pszFilename);
254 
255 
256  bool GetCachedDirList( const char* pszURL,
257  CachedDirList& oCachedDirList );
258  void SetCachedDirList( const char* pszURL,
259  const CachedDirList& oCachedDirList );
260  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
261 
262  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
263 };
264 
265 /************************************************************************/
266 /* VSICurlHandle */
267 /************************************************************************/
268 
269 class VSICurlHandle : public VSIVirtualHandle
270 {
271  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
272 
273  protected:
274  VSICurlFilesystemHandler* poFS = nullptr;
275 
276  bool m_bCached = true;
277 
278  FileProp oFileProp{};
279 
280  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
281  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
282 
283  char **m_papszHTTPOptions = nullptr;
284 
285  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
286  int nBlocksToDownload = 1;
287 
288  bool bStopOnInterruptUntilUninstall = false;
289  bool bInterrupted = false;
290  VSICurlReadCbkFunc pfnReadCbk = nullptr;
291  void *pReadCbkUserData = nullptr;
292 
293  int m_nMaxRetry = 0;
294  double m_dfRetryDelay = 0.0;
295 
296  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
297  const int nBlocks,
298  const char* pBuffer,
299  size_t nSize );
300 
301  private:
302 
303  vsi_l_offset curOffset = 0;
304 
305  bool bEOF = false;
306 
307  virtual bool DownloadRegion(vsi_l_offset startOffset, int nBlocks);
308 
309  bool m_bUseHead = false;
310 
311  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
312  const vsi_l_offset* panOffsets,
313  const size_t* panSizes );
314  CPLString GetRedirectURLIfValid(bool& bHasExpired);
315 
316  protected:
317  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
318  const struct curl_slist* /* psExistingHeaders */)
319  { return nullptr; }
320  virtual bool AllowAutomaticRedirection() { return true; }
321  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
322  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
323  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
324  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
325  void SetURL(const char* pszURL);
326 
327  public:
328 
329  VSICurlHandle( VSICurlFilesystemHandler* poFS,
330  const char* pszFilename,
331  const char* pszURLIn = nullptr );
332  ~VSICurlHandle() override;
333 
334  int Seek( vsi_l_offset nOffset, int nWhence ) override;
335  vsi_l_offset Tell() override;
336  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
337  int ReadMultiRange( int nRanges, void ** ppData,
338  const vsi_l_offset* panOffsets,
339  const size_t* panSizes ) override;
340  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
341  int Eof() override;
342  int Flush() override;
343  int Close() override;
344 
345  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
346  vsi_l_offset GetFileSize() { return GetFileSize(false); }
347  virtual vsi_l_offset GetFileSize( bool bSetError );
348  bool Exists( bool bSetError );
349  bool IsDirectory() const { return oFileProp.bIsDirectory; }
350  time_t GetMTime() const { return oFileProp.mTime; }
351 
352  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
353  void* pfnUserData,
354  int bStopOnInterruptUntilUninstall );
355  int UninstallReadCbk();
356 
357  const char *GetURL() const { return m_pszURL; }
358 };
359 
360 /************************************************************************/
361 /* IVSIS3LikeFSHandler */
362 /************************************************************************/
363 
364 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
365 {
366  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
367 
368  protected:
369  char** GetFileList( const char *pszFilename,
370  int nMaxFiles,
371  bool* pbGotFileList ) override;
372 
373  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
374  const char* pszURI, bool bAllowNoObject) = 0;
375 
376  IVSIS3LikeFSHandler() = default;
377 
378  public:
379  int Unlink( const char *pszFilename ) override;
380  int Mkdir( const char *pszDirname, long nMode ) override;
381  int Rmdir( const char *pszDirname ) override;
382  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
383  int nFlags ) override;
384 
385  virtual int DeleteObject( const char *pszFilename );
386 
387  virtual const char* GetDebugKey() const = 0;
388 
389  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
390  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
391 
392  bool Sync( const char* pszSource, const char* pszTarget,
393  const char* const * papszOptions,
394  GDALProgressFunc pProgressFunc,
395  void *pProgressData,
396  char*** ppapszOutputs ) override;
397 
398  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
399  const char* const *papszOptions) override;
400 };
401 
402 /************************************************************************/
403 /* IVSIS3LikeHandle */
404 /************************************************************************/
405 
406 class IVSIS3LikeHandle: public VSICurlHandle
407 {
408  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
409 
410  protected:
411  bool UseLimitRangeGetInsteadOfHead() override { return true; }
412  bool IsDirectoryFromExists( const char* pszVerb,
413  int response_code ) override
414  {
415  // A bit dirty, but on S3, a GET on a existing directory returns a 416
416  return response_code == 416 && EQUAL(pszVerb, "GET") &&
417  CPLString(m_pszURL).back() == '/';
418  }
419  void ProcessGetFileSizeResult( const char* pszContent ) override
420  {
421  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
422  }
423 
424  public:
425  IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
426  const char* pszFilename,
427  const char* pszURLIn = nullptr ) :
428  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
429  ~IVSIS3LikeHandle() override {}
430 };
431 
432 /************************************************************************/
433 /* VSIS3WriteHandle */
434 /************************************************************************/
435 
436 class VSIS3WriteHandle final : public VSIVirtualHandle
437 {
438  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
439 
440  IVSIS3LikeFSHandler *m_poFS = nullptr;
441  CPLString m_osFilename{};
442  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
443  bool m_bUseChunked = false;
444 
445  vsi_l_offset m_nCurOffset = 0;
446  int m_nBufferOff = 0;
447  int m_nBufferSize = 0;
448  int m_nBufferOffReadCallback = 0;
449  bool m_bClosed = false;
450  GByte *m_pabyBuffer = nullptr;
451  CPLString m_osUploadID{};
452  int m_nPartNumber = 0;
453  std::vector<CPLString> m_aosEtags{};
454  CPLString m_osXML{};
455  int m_nOffsetInXML = 0;
456  bool m_bError = false;
457 
458  CURLM *m_hCurlMulti = nullptr;
459  CURL *m_hCurl = nullptr;
460  const void *m_pBuffer = nullptr;
461  CPLString m_osCurlErrBuf{};
462  size_t m_nChunkedBufferOff = 0;
463  size_t m_nChunkedBufferSize = 0;
464 
465  static size_t ReadCallBackBuffer( char *buffer, size_t size,
466  size_t nitems, void *instream );
467  bool InitiateMultipartUpload();
468  bool UploadPart();
469  static size_t ReadCallBackXML( char *buffer, size_t size,
470  size_t nitems, void *instream );
471  bool CompleteMultipart();
472  bool AbortMultipart();
473  bool DoSinglePartPUT();
474 
475  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
476  size_t nitems, void *instream );
477  size_t WriteChunked( const void *pBuffer,
478  size_t nSize, size_t nMemb );
479  int FinishChunkedTransfer();
480 
481  void InvalidateParentDirectory();
482 
483  public:
484  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
485  const char* pszFilename,
486  IVSIS3LikeHandleHelper* poS3HandleHelper,
487  bool bUseChunked );
488  ~VSIS3WriteHandle() override;
489 
490  int Seek( vsi_l_offset nOffset, int nWhence ) override;
491  vsi_l_offset Tell() override;
492  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
493  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
494  int Eof() override;
495  int Close() override;
496 
497  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
498 };
499 
500 /************************************************************************/
501 /* VSIAppendWriteHandle */
502 /************************************************************************/
503 
504 class VSIAppendWriteHandle : public VSIVirtualHandle
505 {
506  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
507 
508  protected:
509 
510  VSICurlFilesystemHandler* m_poFS = nullptr;
511  CPLString m_osFSPrefix{};
512  CPLString m_osFilename{};
513 
514  vsi_l_offset m_nCurOffset = 0;
515  int m_nBufferOff = 0;
516  int m_nBufferSize = 0;
517  int m_nBufferOffReadCallback = 0;
518  bool m_bClosed = false;
519  GByte *m_pabyBuffer = nullptr;
520  bool m_bError = false;
521 
522  static size_t ReadCallBackBuffer( char *buffer, size_t size,
523  size_t nitems, void *instream );
524  virtual bool Send(bool bIsLastBlock) = 0;
525 
526  public:
527  VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
528  const char* pszFSPrefix,
529  const char* pszFilename,
530  int nChunkSize );
531  virtual ~VSIAppendWriteHandle();
532 
533  int Seek( vsi_l_offset nOffset, int nWhence ) override;
534  vsi_l_offset Tell() override;
535  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
536  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
537  int Eof() override;
538  int Close() override;
539 
540  bool IsOK() { return m_pabyBuffer != nullptr; }
541 };
542 
543 int VSICURLGetDownloadChunkSize();
544 
545 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
546  VSILFILE *fp,
547  VSICurlReadCbkFunc pfnReadCbk,
548  void *pReadCbkUserData );
549 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
550  size_t nmemb, void *req );
551 void MultiPerform(CURLM* hCurlMultiHandle,
552  CURL* hEasyHandle = nullptr);
553 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
554 
555 } // namespace cpl
556 
558 
559 #endif // HAVE_CURL
560 
561 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
Definition: cpl_conv.h:367
Core portability definitions for CPL.
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Convenient string class based on std::string.
Definition: cpl_string.h:329
Various convenience functions for working with strings and string lists.
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:192
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:561
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:438
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:307
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:989
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142

Generated for GDAL by doxygen 1.8.8.