GDAL

gdalalg_abstract_pipeline.h
1/******************************************************************************
2 *
3 * Project: GDAL
4 * Purpose: gdal "raster/vector pipeline" subcommand
5 * Author: Even Rouault <even dot rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13#ifndef GDALALG_ABSTRACT_PIPELINE_INCLUDED
14#define GDALALG_ABSTRACT_PIPELINE_INCLUDED
15
17
18#include "cpl_json.h"
19
20#include "gdalalgorithm.h"
21#include "gdal_priv.h"
22
23#include <algorithm>
24
25// This is an easter egg to pay tribute to PROJ pipeline syntax
26// We accept "gdal vector +gdal=pipeline +step +gdal=read +input=in.tif +step +gdal=reproject +dst-crs=EPSG:32632 +step +gdal=write +output=out.tif +overwrite"
27// as an alternative to (recommended):
28// "gdal vector pipeline ! read in.tif ! reproject--dst-crs=EPSG:32632 ! write out.tif --overwrite"
29#ifndef GDAL_PIPELINE_PROJ_NOSTALGIA
30#define GDAL_PIPELINE_PROJ_NOSTALGIA
31#endif
32
33/************************************************************************/
34/* GDALPipelineStepRunContext */
35/************************************************************************/
36
37 class GDALPipelineStepAlgorithm;
38
39 class GDALPipelineStepRunContext
40{
41 public:
42 GDALPipelineStepRunContext() = default;
43
44 // Progress callback to use during execution of the step
45 GDALProgressFunc m_pfnProgress = nullptr;
46 void *m_pProgressData = nullptr;
47
48 // If there is a step in the pipeline immediately following step to which
49 // this instance of GDALRasterPipelineStepRunContext is passed, and that
50 // this next step is usable by the current step (as determined by
51 // CanHandleNextStep()), then this member will point to this next step.
52 GDALPipelineStepAlgorithm *m_poNextUsableStep = nullptr;
53
54 private:
55 CPL_DISALLOW_COPY_ASSIGN(GDALPipelineStepRunContext)
56};
57
58/************************************************************************/
59/* GDALPipelineStepAlgorithm */
60/************************************************************************/
61
62 class GDALPipelineStepAlgorithm /* non final */ : public GDALAlgorithm
63{
64 public:
65 std::vector<GDALArgDatasetValue> &GetInputDatasets()
66 {
67 return m_inputDataset;
68 }
69
70 const std::vector<GDALArgDatasetValue> &GetInputDatasets() const
71 {
72 return m_inputDataset;
73 }
74
75 GDALArgDatasetValue &GetOutputDataset()
76 {
77 return m_outputDataset;
78 }
79
80 const GDALArgDatasetValue &GetOutputDataset() const
81 {
82 return m_outputDataset;
83 }
84
85 const std::string &GetOutputString() const
86 {
87 return m_output;
88 }
89
90 const std::string &GetOutputLayerName() const
91 {
92 return m_outputLayerName;
93 }
94
95 const std::string &GetOutputFormat() const
96 {
97 return m_format;
98 }
99
100 const std::vector<std::string> &GetCreationOptions() const
101 {
102 return m_creationOptions;
103 }
104
105 const std::vector<std::string> &GetLayerCreationOptions() const
106 {
107 return m_layerCreationOptions;
108 }
109
110 bool GetOverwriteLayer() const
111 {
112 return m_overwriteLayer;
113 }
114
115 bool GetAppendLayer() const
116 {
117 return m_appendLayer;
118 }
119
120 virtual int GetInputType() const = 0;
121
122 virtual int GetOutputType() const = 0;
123
124 bool Finalize() override;
125
126 // Used by GDALDispatcherAlgorithm for vector info/convert
127 GDALDataset *GetInputDatasetRef()
128 {
129 return m_inputDataset.empty() ? nullptr
130 : m_inputDataset[0].GetDatasetRef();
131 }
132
133 // Used by GDALDispatcherAlgorithm for vector info/convert
134 void SetInputDataset(GDALDataset *poDS);
135
136 protected:
137 struct ConstructorOptions
138 {
139 bool standaloneStep = false;
140 bool addDefaultArguments = true;
141 bool autoOpenInputDatasets = true;
142 bool outputDatasetRequired = true;
143 bool addInputLayerNameArgument = true; // only for vector input
144 bool addUpdateArgument = true; // only for vector output
145 bool addAppendLayerArgument = true; // only for vector output
146 bool addOverwriteLayerArgument = true; // only for vector output
147 bool addUpsertArgument = true; // only for vector output
148 bool addSkipErrorsArgument = true; // only for vector output
149 bool addOutputLayerNameArgument = true; // only for vector output
150 int inputDatasetMaxCount = 1;
151 std::string inputDatasetHelpMsg{};
152 std::string inputDatasetAlias{};
153 std::string inputDatasetMetaVar = "INPUT";
154 std::string outputDatasetHelpMsg{};
155 std::string outputFormatCreateCapability = GDAL_DCAP_CREATECOPY;
156
157 inline ConstructorOptions &SetStandaloneStep(bool b)
158 {
159 standaloneStep = b;
160 return *this;
161 }
162
163 inline ConstructorOptions &SetAddDefaultArguments(bool b)
164 {
165 addDefaultArguments = b;
166 return *this;
167 }
168
169 inline ConstructorOptions &SetAddInputLayerNameArgument(bool b)
170 {
171 addInputLayerNameArgument = b;
172 return *this;
173 }
174
175 inline ConstructorOptions &SetInputDatasetMaxCount(int maxCount)
176 {
177 inputDatasetMaxCount = maxCount;
178 return *this;
179 }
180
181 inline ConstructorOptions &SetInputDatasetHelpMsg(const std::string &s)
182 {
183 inputDatasetHelpMsg = s;
184 return *this;
185 }
186
187 inline ConstructorOptions &SetInputDatasetAlias(const std::string &s)
188 {
189 inputDatasetAlias = s;
190 return *this;
191 }
192
193 inline ConstructorOptions &SetInputDatasetMetaVar(const std::string &s)
194 {
195 inputDatasetMetaVar = s;
196 return *this;
197 }
198
199 inline ConstructorOptions &SetOutputDatasetHelpMsg(const std::string &s)
200 {
201 outputDatasetHelpMsg = s;
202 return *this;
203 }
204
205 inline ConstructorOptions &SetAutoOpenInputDatasets(bool b)
206 {
207 autoOpenInputDatasets = b;
208 return *this;
209 }
210
211 inline ConstructorOptions &SetOutputDatasetRequired(bool b)
212 {
213 outputDatasetRequired = b;
214 return *this;
215 }
216
217 inline ConstructorOptions &
218 SetOutputFormatCreateCapability(const std::string &capability)
219 {
220 outputFormatCreateCapability = capability;
221 return *this;
222 }
223
224 inline ConstructorOptions &SetAddAppendLayerArgument(bool b)
225 {
226 addAppendLayerArgument = b;
227 return *this;
228 }
229
230 inline ConstructorOptions &SetAddOverwriteLayerArgument(bool b)
231 {
232 addOverwriteLayerArgument = b;
233 return *this;
234 }
235
236 inline ConstructorOptions &SetAddUpdateArgument(bool b)
237 {
238 addUpdateArgument = b;
239 return *this;
240 }
241
242 inline ConstructorOptions &SetAddUpsertArgument(bool b)
243 {
244 addUpsertArgument = b;
245 return *this;
246 }
247
248 inline ConstructorOptions &SetAddSkipErrorsArgument(bool b)
249 {
250 addSkipErrorsArgument = b;
251 return *this;
252 }
253
254 inline ConstructorOptions &SetAddOutputLayerNameArgument(bool b)
255 {
256 addOutputLayerNameArgument = b;
257 return *this;
258 }
259 };
260
261 GDALPipelineStepAlgorithm(const std::string &name,
262 const std::string &description,
263 const std::string &helpURL,
264 const ConstructorOptions &);
265
266 friend class GDALPipelineAlgorithm;
267 friend class GDALRasterPipelineAlgorithm;
268 friend class GDALVectorPipelineAlgorithm;
269 friend class GDALAbstractPipelineAlgorithm;
270
271 virtual bool CanBeFirstStep() const
272 {
273 return false;
274 }
275
276 virtual bool CanBeMiddleStep() const
277 {
278 return !CanBeFirstStep() && !CanBeLastStep();
279 }
280
281 virtual bool CanBeLastStep() const
282 {
283 return false;
284 }
285
287 virtual bool GeneratesFilesFromUserInput() const
288 {
289 return false;
290 }
291
292 virtual bool IsNativelyStreamingCompatible() const
293 {
294 return true;
295 }
296
297 virtual bool SupportsInputMultiThreading() const
298 {
299 return false;
300 }
301
302 virtual bool CanHandleNextStep(GDALPipelineStepAlgorithm *) const
303 {
304 return false;
305 }
306
307 virtual CPLJSONObject Get_OGR_SCHEMA_OpenOption_Layer() const
308 {
309 CPLJSONObject obj;
310 obj.Deinit();
311 return obj;
312 }
313
314 virtual bool RunStep(GDALPipelineStepRunContext &ctxt) = 0;
315
316 bool m_standaloneStep = false;
317 const ConstructorOptions m_constructorOptions;
318 bool m_outputVRTCompatible = true;
319 std::string m_helpDocCategory{};
320
321 // Input arguments
322 std::vector<GDALArgDatasetValue> m_inputDataset{};
323 std::vector<std::string> m_openOptions{};
324 std::vector<std::string> m_inputFormats{};
325 std::vector<std::string> m_inputLayerNames{};
326
327 // Output arguments
328 bool m_stdout = false;
329 std::string m_output{};
330 GDALArgDatasetValue m_outputDataset{};
331 std::string m_format{};
332 std::vector<std::string> m_outputOpenOptions{};
333 std::vector<std::string> m_creationOptions{};
334 bool m_overwrite = false;
335 std::string m_outputLayerName{};
336 GDALInConstructionAlgorithmArg *m_outputFormatArg = nullptr;
337 bool m_appendRaster = false;
338
339 // Output arguments (vector specific)
340 std::vector<std::string> m_layerCreationOptions{};
341 bool m_update = false;
342 bool m_overwriteLayer = false;
343 bool m_appendLayer = false;
344 bool m_upsert = false;
345 bool m_skipErrors = false;
346
347 void AddRasterInputArgs(bool openForMixedRasterVector, bool hiddenForCLI);
348 void AddRasterOutputArgs(bool hiddenForCLI);
349 void AddRasterHiddenInputDatasetArg();
350
351 void AddVectorInputArgs(bool hiddenForCLI);
352 void AddVectorOutputArgs(bool hiddenForCLI,
353 bool shortNameOutputLayerAllowed);
354 using GDALAlgorithm::AddOutputLayerNameArg;
355 void AddOutputLayerNameArg(bool hiddenForCLI,
356 bool shortNameOutputLayerAllowed);
357
358 private:
359 bool RunImpl(GDALProgressFunc pfnProgress, void *pProgressData) override;
360 GDALAlgorithm::ProcessGDALGOutputRet ProcessGDALGOutput() override;
361 bool CheckSafeForStreamOutput() override;
362
363 CPL_DISALLOW_COPY_ASSIGN(GDALPipelineStepAlgorithm)
364};
365
366/************************************************************************/
367/* GDALAbstractPipelineAlgorithm */
368/************************************************************************/
369
370 class GDALAbstractPipelineAlgorithm CPL_NON_FINAL
371 : public GDALPipelineStepAlgorithm
372{
373 public:
374 std::vector<std::string> GetAutoComplete(std::vector<std::string> &args,
375 bool lastWordIsComplete,
376 bool /* showAllOptions*/) override;
377
378 bool Finalize() override;
379
380 std::string GetUsageAsJSON() const override;
381
382 bool
383 ParseCommandLineArguments(const std::vector<std::string> &args) override;
384
385 bool HasSteps() const
386 {
387 return !m_steps.empty();
388 }
389
390 static constexpr const char *OPEN_NESTED_PIPELINE = "[";
391 static constexpr const char *CLOSE_NESTED_PIPELINE = "]";
392
393 static constexpr const char *RASTER_SUFFIX = "-raster";
394 static constexpr const char *VECTOR_SUFFIX = "-vector";
395
396 protected:
397 friend class GDALTeeStepAlgorithmAbstract;
398
399 GDALAbstractPipelineAlgorithm(
400 const std::string &name, const std::string &description,
401 const std::string &helpURL,
402 const GDALPipelineStepAlgorithm::ConstructorOptions &options)
403 : GDALPipelineStepAlgorithm(
404 name, description, helpURL,
405 ConstructorOptions(options).SetAutoOpenInputDatasets(false))
406 {
407 }
408
409 std::string m_pipeline{};
410
411 virtual GDALAlgorithmRegistry &GetStepRegistry() = 0;
412
413 virtual const GDALAlgorithmRegistry &GetStepRegistry() const = 0;
414
415 std::unique_ptr<GDALPipelineStepAlgorithm>
416 GetStepAlg(const std::string &name) const;
417
418 bool HasOutputString() const override;
419
420 static bool IsReadSpecificArgument(const char *pszArgName);
421 static bool IsWriteSpecificArgument(const char *pszArgName);
422
423 private:
424 friend class GDALPipelineAlgorithm;
425 friend class GDALRasterPipelineAlgorithm;
426 friend class GDALVectorPipelineAlgorithm;
427
428 std::vector<std::unique_ptr<GDALPipelineStepAlgorithm>> m_steps{};
429
430 std::unique_ptr<GDALPipelineStepAlgorithm> m_stepOnWhichHelpIsRequested{};
431
432 bool m_bInnerPipeline = false;
433 bool m_bExpectReadStep = true;
434
435 enum class StepConstraint
436 {
437 MUST_BE,
438 CAN_BE,
439 CAN_NOT_BE
440 };
441
442 StepConstraint m_eLastStepAsWrite = StepConstraint::CAN_BE;
443
444 std::vector<std::unique_ptr<GDALAbstractPipelineAlgorithm>>
445 m_apoNestedPipelines{};
446
447 // More would lead to unreadable pipelines
448 static constexpr int MAX_NESTING_LEVEL = 3;
449
450 bool
451 CheckFirstAndLastStep(const std::vector<GDALPipelineStepAlgorithm *> &steps,
452 bool forAutoComplete) const;
453
454 bool ParseCommandLineArguments(const std::vector<std::string> &args,
455 bool forAutoComplete);
456
457 bool RunStep(GDALPipelineStepRunContext &ctxt) override;
458
459 std::string
460 BuildNestedPipeline(GDALPipelineStepAlgorithm *curAlg,
461 std::vector<std::string> &nestedPipelineArgs,
462 bool forAutoComplete);
463
464 bool SaveGDALGFile(const std::string &outFilename,
465 std::string &outString) const;
466
467 virtual std::unique_ptr<GDALAbstractPipelineAlgorithm>
468 CreateNestedPipeline() const = 0;
469};
470
472
473#endif
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition cpl_json.h:56
CPLJSONObject::Deinit
void Deinit()
Decrement reference counter and make pointer NULL.
Definition cpl_json.cpp:1343
GDALAlgorithmRegistry
Registry of GDAL algorithms.
Definition gdalalgorithm_cpp.h:2132
GDALAlgorithm
GDAL algorithm.
Definition gdalalgorithm_cpp.h:2261
GDALAlgorithm::Finalize
virtual bool Finalize()
Complete any pending actions, and return the final status.
Definition gdalalgorithm.cpp:5819
GDALAlgorithm::ProcessGDALGOutputRet
ProcessGDALGOutputRet
Return value for ProcessGDALGOutput.
Definition gdalalgorithm_cpp.h:2926
GDALAlgorithm::AddOutputLayerNameArg
GDALInConstructionAlgorithmArg & AddOutputLayerNameArg(std::string *pValue, const char *helpMessage=nullptr)
Add (single) output layer name argument.
Definition gdalalgorithm.cpp:4665
GDALAlgorithm::CheckSafeForStreamOutput
virtual bool CheckSafeForStreamOutput()
Method executed by Run() when m_executionForStreamOutput is set to ensure the command is safe to exec...
Definition gdalalgorithm.cpp:5796
GDALAlgorithm::ProcessGDALGOutput
virtual ProcessGDALGOutputRet ProcessGDALGOutput()
Process output to a .gdalg file.
Definition gdalalgorithm.cpp:5232
GDALArgDatasetValue
Value for an argument that points to a GDALDataset.
Definition gdalalgorithm_cpp.h:149
GDALDataset
A set of associated raster bands, usually from one file.
Definition gdal_dataset.h:76
GDALInConstructionAlgorithmArg
Technical class used by GDALAlgorithm when constructing argument declarations.
Definition gdalalgorithm_cpp.h:1771
cpl_json.h
Interface for read and write JSON documents.
CPL_NON_FINAL
#define CPL_NON_FINAL
Mark that a class is explicitly recognized as non-final.
Definition cpl_port.h:929
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition cpl_port.h:936
GDAL_DCAP_CREATECOPY
#define GDAL_DCAP_CREATECOPY
Capability set by a driver that implements the CreateCopy() API.
Definition gdal.h:597
gdal_priv.h
This file is legacy since GDAL 3.12, but will be kept at least in the whole GDAL 3....