GDAL

cpl_vsi_virtual.h
1/******************************************************************************
2 * $Id$
3 *
4 * Project: VSI Virtual File System
5 * Purpose: Declarations for classes related to the virtual filesystem.
6 * These would only be normally required by applications implementing
7 * their own virtual file system classes which should be rare.
8 * The class interface may be fragile through versions.
9 * Author: Frank Warmerdam, warmerdam@pobox.com
10 *
11 ******************************************************************************
12 * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
13 * Copyright (c) 2010-2014, Even Rouault <even dot rouault at spatialys.com>
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a
16 * copy of this software and associated documentation files (the "Software"),
17 * to deal in the Software without restriction, including without limitation
18 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
19 * and/or sell copies of the Software, and to permit persons to whom the
20 * Software is furnished to do so, subject to the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included
23 * in all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
26 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
28 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
31 * DEALINGS IN THE SOFTWARE.
32 ****************************************************************************/
33
34#ifndef CPL_VSI_VIRTUAL_H_INCLUDED
35#define CPL_VSI_VIRTUAL_H_INCLUDED
36
37#include "cpl_vsi.h"
38#include "cpl_vsi_error.h"
39#include "cpl_string.h"
40#include "cpl_multiproc.h"
41
42#include <map>
43#include <memory>
44#include <vector>
45#include <string>
46
47// To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
48#ifdef GetDiskFreeSpace
49#undef GetDiskFreeSpace
50#endif
51
52// To avoid aliasing to CopyFile to CopyFileA on Windows
53#ifdef CopyFile
54#undef CopyFile
55#endif
56
57/************************************************************************/
58/* VSIVirtualHandle */
59/************************************************************************/
60
62 struct CPL_DLL VSIVirtualHandle
63{
64 public:
65 virtual int Seek(vsi_l_offset nOffset, int nWhence) = 0;
66 virtual vsi_l_offset Tell() = 0;
67 virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) = 0;
68 virtual int ReadMultiRange(int nRanges, void **ppData,
69 const vsi_l_offset *panOffsets,
70 const size_t *panSizes);
71
84 virtual void AdviseRead(CPL_UNUSED int nRanges,
85 CPL_UNUSED const vsi_l_offset *panOffsets,
86 CPL_UNUSED const size_t *panSizes)
87 {
88 }
89
102 virtual size_t GetAdviseReadTotalBytesLimit() const
103 {
104 return 0;
105 }
106
107 virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount) = 0;
108
109 int Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
110 CPL_PRINT_FUNC_FORMAT(2, 3);
111
112 virtual int Eof() = 0;
113
114 virtual int Flush()
115 {
116 return 0;
117 }
118
119 virtual int Close() = 0;
120 // Base implementation that only supports file extension.
121 virtual int Truncate(vsi_l_offset nNewSize);
122
123 virtual void *GetNativeFileDescriptor()
124 {
125 return nullptr;
126 }
127
128 virtual VSIRangeStatus GetRangeStatus(CPL_UNUSED vsi_l_offset nOffset,
129 CPL_UNUSED vsi_l_offset nLength)
130 {
131 return VSI_RANGE_STATUS_UNKNOWN;
132 }
133
134 virtual bool HasPRead() const;
135 virtual size_t PRead(void *pBuffer, size_t nSize,
136 vsi_l_offset nOffset) const;
137
138 // NOTE: when adding new methods, besides the "actual" implementations,
139 // also consider the VSICachedFile one.
140
141 virtual ~VSIVirtualHandle()
142 {
143 }
144};
145
146/************************************************************************/
147/* VSIVirtualHandleCloser */
148/************************************************************************/
149
152 struct VSIVirtualHandleCloser
153{
155 void operator()(VSIVirtualHandle *poHandle)
156 {
157 if (poHandle)
158 {
159 poHandle->Close();
160 delete poHandle;
161 }
162 }
163};
164
166 typedef std::unique_ptr<VSIVirtualHandle, VSIVirtualHandleCloser>
167 VSIVirtualHandleUniquePtr;
168
169/************************************************************************/
170/* VSIFilesystemHandler */
171/************************************************************************/
172
173#ifndef DOXYGEN_SKIP
174 class CPL_DLL VSIFilesystemHandler
175{
176
177 public:
178 virtual ~VSIFilesystemHandler()
179 {
180 }
181
182 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess);
183
184 virtual VSIVirtualHandle *Open(const char *pszFilename,
185 const char *pszAccess, bool bSetError,
186 CSLConstList papszOptions) = 0;
187 virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
188 int nFlags) = 0;
189
190 virtual int Unlink(const char *pszFilename)
191 {
192 (void)pszFilename;
193 errno = ENOENT;
194 return -1;
195 }
196
197 virtual int *UnlinkBatch(CSLConstList papszFiles);
198
199 virtual int Mkdir(const char *pszDirname, long nMode)
200 {
201 (void)pszDirname;
202 (void)nMode;
203 errno = ENOENT;
204 return -1;
205 }
206
207 virtual int Rmdir(const char *pszDirname)
208 {
209 (void)pszDirname;
210 errno = ENOENT;
211 return -1;
212 }
213
214 virtual int RmdirRecursive(const char *pszDirname);
215
216 char **ReadDir(const char *pszDirname)
217 {
218 return ReadDirEx(pszDirname, 0);
219 }
220
221 virtual char **ReadDirEx(const char * /*pszDirname*/, int /* nMaxFiles */)
222 {
223 return nullptr;
224 }
225
226 virtual char **SiblingFiles(const char * /*pszFilename*/)
227 {
228 return nullptr;
229 }
230
231 virtual int Rename(const char *oldpath, const char *newpath)
232 {
233 (void)oldpath;
234 (void)newpath;
235 errno = ENOENT;
236 return -1;
237 }
238
239 virtual int IsCaseSensitive(const char *pszFilename)
240 {
241 (void)pszFilename;
242 return TRUE;
243 }
244
245 virtual GIntBig GetDiskFreeSpace(const char * /* pszDirname */)
246 {
247 return -1;
248 }
249
250 virtual int SupportsSparseFiles(const char * /* pszPath */)
251 {
252 return FALSE;
253 }
254
255 virtual int HasOptimizedReadMultiRange(const char * /* pszPath */)
256 {
257 return FALSE;
258 }
259
260 virtual const char *GetActualURL(const char * /*pszFilename*/)
261 {
262 return nullptr;
263 }
264
265 virtual const char *GetOptions()
266 {
267 return nullptr;
268 }
269
270 virtual char *GetSignedURL(const char * /*pszFilename*/,
271 CSLConstList /* papszOptions */)
272 {
273 return nullptr;
274 }
275
276 virtual bool Sync(const char *pszSource, const char *pszTarget,
277 const char *const *papszOptions,
278 GDALProgressFunc pProgressFunc, void *pProgressData,
279 char ***ppapszOutputs);
280
281 virtual int CopyFile(const char *pszSource, const char *pszTarget,
282 VSILFILE *fpSource, vsi_l_offset nSourceSize,
283 const char *const *papszOptions,
284 GDALProgressFunc pProgressFunc, void *pProgressData);
285
286 virtual VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
287 const char *const *papszOptions);
288
289 virtual char **GetFileMetadata(const char *pszFilename,
290 const char *pszDomain,
291 CSLConstList papszOptions);
292
293 virtual bool SetFileMetadata(const char *pszFilename,
294 CSLConstList papszMetadata,
295 const char *pszDomain,
296 CSLConstList papszOptions);
297
298 virtual bool AbortPendingUploads(const char * /*pszFilename*/)
299 {
300 return true;
301 }
302
303 virtual std::string
304 GetStreamingFilename(const std::string &osFilename) const
305 {
306 return osFilename;
307 }
308
316 virtual std::string
317 GetCanonicalFilename(const std::string &osFilename) const
318 {
319 return osFilename;
320 }
321
322 virtual bool IsLocal(const char * /* pszPath */)
323 {
324 return true;
325 }
326
327 virtual bool SupportsSequentialWrite(const char * /* pszPath */,
328 bool /* bAllowLocalTempFile */)
329 {
330 return true;
331 }
332
333 virtual bool SupportsRandomWrite(const char * /* pszPath */,
334 bool /* bAllowLocalTempFile */)
335 {
336 return true;
337 }
338
339 virtual bool SupportsRead(const char * /* pszPath */)
340 {
341 return true;
342 }
343
344 virtual VSIFilesystemHandler *Duplicate(const char * /* pszPrefix */)
345 {
346 CPLError(CE_Failure, CPLE_NotSupported,
347 "Duplicate() not supported on this file system");
348 return nullptr;
349 }
350
357 virtual const char *GetDirectorySeparator(CPL_UNUSED const char *pszPath)
358 {
359 return "/";
360 }
361};
362#endif /* #ifndef DOXYGEN_SKIP */
363
364/************************************************************************/
365/* VSIFileManager */
366/************************************************************************/
367
368#ifndef DOXYGEN_SKIP
369 class CPL_DLL VSIFileManager
370{
371 private:
372 VSIFilesystemHandler *poDefaultHandler = nullptr;
373 std::map<std::string, VSIFilesystemHandler *> oHandlers{};
374
375 VSIFileManager();
376
377 static VSIFileManager *Get();
378
379 CPL_DISALLOW_COPY_ASSIGN(VSIFileManager)
380
381 public:
382 ~VSIFileManager();
383
384 static VSIFilesystemHandler *GetHandler(const char *);
385 static void InstallHandler(const std::string &osPrefix,
386 VSIFilesystemHandler *);
387 static void RemoveHandler(const std::string &osPrefix);
388
389 static char **GetPrefixes();
390};
391#endif /* #ifndef DOXYGEN_SKIP */
392
393/************************************************************************/
394/* ==================================================================== */
395/* VSIArchiveFilesystemHandler */
396/* ==================================================================== */
397/************************************************************************/
398
399#ifndef DOXYGEN_SKIP
400
401 class VSIArchiveEntryFileOffset
402{
403 public:
404 virtual ~VSIArchiveEntryFileOffset();
405};
406
407 typedef struct
408{
409 char *fileName;
410 vsi_l_offset uncompressed_size;
411 VSIArchiveEntryFileOffset *file_pos;
412 int bIsDir;
413 GIntBig nModifiedTime;
414} VSIArchiveEntry;
415
416 class VSIArchiveContent
417{
418 public:
419 time_t mTime = 0;
420 vsi_l_offset nFileSize = 0;
421 int nEntries = 0;
422 VSIArchiveEntry *entries = nullptr;
423
424 ~VSIArchiveContent();
425};
426
427 class VSIArchiveReader
428{
429 public:
430 virtual ~VSIArchiveReader();
431
432 virtual int GotoFirstFile() = 0;
433 virtual int GotoNextFile() = 0;
434 virtual VSIArchiveEntryFileOffset *GetFileOffset() = 0;
435 virtual GUIntBig GetFileSize() = 0;
436 virtual CPLString GetFileName() = 0;
437 virtual GIntBig GetModifiedTime() = 0;
438 virtual int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) = 0;
439};
440
441 class VSIArchiveFilesystemHandler : public VSIFilesystemHandler
442{
443 CPL_DISALLOW_COPY_ASSIGN(VSIArchiveFilesystemHandler)
444
445 protected:
446 CPLMutex *hMutex = nullptr;
447 /* We use a cache that contains the list of files contained in a VSIArchive
448 * file as */
449 /* unarchive.c is quite inefficient in listing them. This speeds up access
450 * to VSIArchive files */
451 /* containing ~1000 files like a CADRG product */
452 std::map<CPLString, VSIArchiveContent *> oFileList{};
453
454 virtual const char *GetPrefix() = 0;
455 virtual std::vector<CPLString> GetExtensions() = 0;
456 virtual VSIArchiveReader *CreateReader(const char *pszArchiveFileName) = 0;
457
458 public:
459 VSIArchiveFilesystemHandler();
460 virtual ~VSIArchiveFilesystemHandler();
461
462 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
463 int nFlags) override;
464 int Unlink(const char *pszFilename) override;
465 int Rename(const char *oldpath, const char *newpath) override;
466 int Mkdir(const char *pszDirname, long nMode) override;
467 int Rmdir(const char *pszDirname) override;
468 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
469
470 virtual const VSIArchiveContent *
471 GetContentOfArchive(const char *archiveFilename,
472 VSIArchiveReader *poReader = nullptr);
473 virtual char *SplitFilename(const char *pszFilename,
474 CPLString &osFileInArchive,
475 int bCheckMainFileExists);
476 virtual VSIArchiveReader *OpenArchiveFile(const char *archiveFilename,
477 const char *fileInArchiveName);
478 virtual int FindFileInArchive(const char *archiveFilename,
479 const char *fileInArchiveName,
480 const VSIArchiveEntry **archiveEntry);
481
482 virtual bool IsLocal(const char *pszPath) override;
483
484 virtual bool
485 SupportsSequentialWrite(const char * /* pszPath */,
486 bool /* bAllowLocalTempFile */) override
487 {
488 return false;
489 }
490
491 virtual bool SupportsRandomWrite(const char * /* pszPath */,
492 bool /* bAllowLocalTempFile */) override
493 {
494 return false;
495 }
496};
497
498/************************************************************************/
499/* VSIDIR */
500/************************************************************************/
501
502 struct CPL_DLL VSIDIR
503{
504 VSIDIR() = default;
505 virtual ~VSIDIR();
506
507 virtual const VSIDIREntry *NextDirEntry() = 0;
508
509 private:
510 VSIDIR(const VSIDIR &) = delete;
511 VSIDIR &operator=(const VSIDIR &) = delete;
512};
513
514#endif /* #ifndef DOXYGEN_SKIP */
515
516 VSIVirtualHandle CPL_DLL *
517VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
518 VSIVirtualHandle *
519VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
520 const GByte *pabyBeginningContent,
521 vsi_l_offset nCheatFileSize);
522 constexpr int VSI_CACHED_DEFAULT_CHUNK_SIZE = 32768;
523 VSIVirtualHandle CPL_DLL *
524VSICreateCachedFile(VSIVirtualHandle *poBaseHandle,
525 size_t nChunkSize = VSI_CACHED_DEFAULT_CHUNK_SIZE,
526 size_t nCacheSize = 0);
527
528 const int CPL_DEFLATE_TYPE_GZIP = 0;
529 const int CPL_DEFLATE_TYPE_ZLIB = 1;
530 const int CPL_DEFLATE_TYPE_RAW_DEFLATE = 2;
531 VSIVirtualHandle CPL_DLL *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
532 int nDeflateType,
533 int bAutoCloseBaseHandle);
534
535 VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
536 int nDeflateType,
537 bool bAutoCloseBaseHandle, int nThreads,
538 size_t nChunkSize,
539 size_t nSOZIPIndexEltSize,
540 std::vector<uint8_t> *panSOZIPIndex);
541
542 VSIVirtualHandle *
543VSICreateUploadOnCloseFile(VSIVirtualHandleUniquePtr &&poWritableHandle,
544 VSIVirtualHandleUniquePtr &&poTmpFile,
545 const std::string &osTmpFilename);
546
547#endif /* ndef CPL_VSI_VIRTUAL_H_INCLUDED */
CPLString
Convenient string class based on std::string.
Definition cpl_string.h:320
CPLE_NotSupported
#define CPLE_NotSupported
Not supported.
Definition cpl_error.h:110
GUIntBig
unsigned long long GUIntBig
Large unsigned integer type (generally 64-bit unsigned integer type).
Definition cpl_port.h:218
CPL_UNUSED
#define CPL_UNUSED
Qualifier for an argument that is unused.
Definition cpl_port.h:985
CPL_FORMAT_STRING
#define CPL_FORMAT_STRING(arg)
Macro into which to wrap the format argument of a printf-like function.
Definition cpl_port.h:966
CPL_PRINT_FUNC_FORMAT
#define CPL_PRINT_FUNC_FORMAT(format_idx, arg_idx)
Tag a function to have printf() formatting.
Definition cpl_port.h:950
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:1042
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition cpl_port.h:1183
GByte
unsigned char GByte
Unsigned byte type.
Definition cpl_port.h:185
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition cpl_port.h:215
cpl_string.h
Various convenience functions for working with strings and string lists.
cpl_vsi.h
Standard C Covers.
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition cpl_vsi.h:203
VSIRangeStatus
VSIRangeStatus
Range status.
Definition cpl_vsi.h:182
VSI_RANGE_STATUS_UNKNOWN
@ VSI_RANGE_STATUS_UNKNOWN
Unknown.
Definition cpl_vsi.h:183
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition cpl_vsi.h:402
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition cpl_vsi.h:146
VSIDIREntry
Directory entry.
Definition cpl_vsi.h:414
VSIVirtualHandleCloser
Helper close to use with a std:unique_ptr<VSIVirtualHandle>, such as VSIVirtualHandleUniquePtr.
Definition cpl_vsi_virtual.h:153
VSIVirtualHandleCloser::operator()
void operator()(VSIVirtualHandle *poHandle)
Operator () that closes and deletes the file handle.
Definition cpl_vsi_virtual.h:155
VSIVirtualHandle
Virtual file handle.
Definition cpl_vsi_virtual.h:63
VSIVirtualHandle::Close
virtual int Close()=0
Close file.
VSIVirtualHandle::GetRangeStatus
virtual VSIRangeStatus GetRangeStatus(vsi_l_offset nOffset, vsi_l_offset nLength)
Return if a given file range contains data or holes filled with zeroes.
Definition cpl_vsi_virtual.h:128
VSIVirtualHandle::Tell
virtual vsi_l_offset Tell()=0
Tell current file offset.
VSIVirtualHandle::Seek
virtual int Seek(vsi_l_offset nOffset, int nWhence)=0
Seek to requested offset.
VSIVirtualHandle::Read
virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount)=0
Read bytes from file.
VSIVirtualHandle::GetNativeFileDescriptor
virtual void * GetNativeFileDescriptor()
Returns the "native" file descriptor for the virtual handle.
Definition cpl_vsi_virtual.h:123
VSIVirtualHandle::GetAdviseReadTotalBytesLimit
virtual size_t GetAdviseReadTotalBytesLimit() const
Return the total maximum number of bytes that AdviseRead() can handle at once.
Definition cpl_vsi_virtual.h:102
VSIVirtualHandle::AdviseRead
virtual void AdviseRead(int nRanges, const vsi_l_offset *panOffsets, const size_t *panSizes)
This method is called when code plans to access soon one or several ranges in a file.
Definition cpl_vsi_virtual.h:84
VSIVirtualHandle::ReadMultiRange
virtual int ReadMultiRange(int nRanges, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes)
Read several ranges of bytes from file.
VSIVirtualHandle::Write
virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount)=0
Write bytes to file.
VSIVirtualHandle::Truncate
virtual int Truncate(vsi_l_offset nNewSize)
Truncate/expand the file to the specified size.