openPMD-api
IOTask.hpp
1 /* Copyright 2017-2021 Fabian Koller
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/ChunkInfo.hpp"
24 #include "openPMD/Dataset.hpp"
25 #include "openPMD/IterationEncoding.hpp"
26 #include "openPMD/Streaming.hpp"
27 #include "openPMD/auxiliary/Export.hpp"
28 #include "openPMD/auxiliary/Memory.hpp"
29 #include "openPMD/auxiliary/Variant.hpp"
30 #include "openPMD/backend/Attribute.hpp"
31 #include "openPMD/backend/ParsePreference.hpp"
32 
33 #include <cstddef>
34 #include <map>
35 #include <memory>
36 #include <string>
37 #include <utility>
38 #include <variant>
39 #include <vector>
40 
41 namespace openPMD
42 {
43 class Attributable;
44 class Writable;
45 
46 Writable *getWritable(Attributable *);
47 
51  CREATE_FILE, CHECK_FILE, OPEN_FILE, CLOSE_FILE,
52  DELETE_FILE,
53 
54  CREATE_PATH, CLOSE_PATH, OPEN_PATH, DELETE_PATH,
55  LIST_PATHS,
56 
57  CREATE_DATASET, EXTEND_DATASET, OPEN_DATASET, DELETE_DATASET,
58  WRITE_DATASET, READ_DATASET, LIST_DATASETS, GET_BUFFER_VIEW,
59 
60  DELETE_ATT, WRITE_ATT, READ_ATT, LIST_ATTS,
61 
62  ADVANCE,
63  AVAILABLE_CHUNKS,
64  DEREGISTER
65 }; // note: if you change the enum members here, please update
66  // docs/source/dev/design.rst
67 
68 namespace internal
69 {
70  /*
71  * The returned strings are compile-time constants, so no worries about
72  * pointer validity.
73  */
74  OPENPMDAPI_EXPORT std::string operationAsString(Operation);
75 } // namespace internal
76 
77 struct OPENPMDAPI_EXPORT AbstractParameter
78 {
79  virtual ~AbstractParameter() = default;
80  AbstractParameter() = default;
81 
82  virtual std::unique_ptr<AbstractParameter> to_heap() && = 0;
83 
84 protected:
85  // avoid object slicing
86  // by allow only child classes to use these things for defining their own
87  // copy/move constructors/assignment operators
88  AbstractParameter(const AbstractParameter &) = default;
89  AbstractParameter &operator=(const AbstractParameter &) = default;
91  AbstractParameter &operator=(AbstractParameter &&) = default;
92 };
93 
103 template <Operation>
104 struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter
105 {
106  Parameter() = delete;
107  Parameter(Parameter const &) = delete;
108  Parameter(Parameter &&) = delete;
109 };
110 
111 template <>
112 struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_FILE>
113  : public AbstractParameter
114 {
115  Parameter() = default;
116  Parameter(Parameter &&) = default;
117  Parameter(Parameter const &) = default;
118  Parameter &operator=(Parameter &&) = default;
119  Parameter &operator=(Parameter const &) = default;
120 
121  std::unique_ptr<AbstractParameter> to_heap() && override
122  {
123  return std::unique_ptr<AbstractParameter>(
124  new Parameter<Operation::CREATE_FILE>(std::move(*this)));
125  }
126 
127  std::string name = "";
128 };
129 
130 template <>
131 struct OPENPMDAPI_EXPORT Parameter<Operation::CHECK_FILE>
132  : public AbstractParameter
133 {
134  Parameter() = default;
135  Parameter(Parameter &&) = default;
136  Parameter(Parameter const &) = default;
137  Parameter &operator=(Parameter &&) = default;
138  Parameter &operator=(Parameter const &) = default;
139 
140  std::unique_ptr<AbstractParameter> to_heap() && override
141  {
142  return std::unique_ptr<AbstractParameter>(
143  new Parameter<Operation::CHECK_FILE>(std::move(*this)));
144  }
145 
146  std::string name = "";
147  enum class FileExists
148  {
149  DontKnow,
150  Yes,
151  No
152  };
153  std::shared_ptr<FileExists> fileExists =
154  std::make_shared<FileExists>(FileExists::DontKnow);
155 };
156 
157 template <>
158 struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_FILE>
159  : public AbstractParameter
160 {
161  Parameter() = default;
162  Parameter(Parameter &&) = default;
163  Parameter(Parameter const &) = default;
164  Parameter &operator=(Parameter &&) = default;
165  Parameter &operator=(Parameter const &) = default;
166 
167  std::unique_ptr<AbstractParameter> to_heap() && override
168  {
169  return std::unique_ptr<AbstractParameter>(
170  new Parameter<Operation::OPEN_FILE>(std::move(*this)));
171  }
172 
173  std::string name = "";
174  using ParsePreference = internal::ParsePreference;
175  std::shared_ptr<ParsePreference> out_parsePreference =
176  std::make_shared<ParsePreference>(ParsePreference::UpFront);
177 };
178 
179 template <>
180 struct OPENPMDAPI_EXPORT Parameter<Operation::CLOSE_FILE>
181  : public AbstractParameter
182 {
183  Parameter() = default;
184  Parameter(Parameter &&) = default;
185  Parameter(Parameter const &) = default;
186  Parameter &operator=(Parameter &&) = default;
187  Parameter &operator=(Parameter const &) = default;
188 
189  std::unique_ptr<AbstractParameter> to_heap() && override
190  {
191  return std::unique_ptr<AbstractParameter>(
192  new Parameter<Operation::CLOSE_FILE>(std::move(*this)));
193  }
194 };
195 
196 template <>
197 struct OPENPMDAPI_EXPORT Parameter<Operation::DELETE_FILE>
198  : public AbstractParameter
199 {
200  Parameter() = default;
201  Parameter(Parameter &&) = default;
202  Parameter(Parameter const &) = default;
203  Parameter &operator=(Parameter &&) = default;
204  Parameter &operator=(Parameter const &) = default;
205 
206  std::unique_ptr<AbstractParameter> to_heap() && override
207  {
208  return std::unique_ptr<AbstractParameter>(
209  new Parameter<Operation::DELETE_FILE>(std::move(*this)));
210  }
211 
212  std::string name = "";
213 };
214 
215 template <>
216 struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_PATH>
217  : public AbstractParameter
218 {
219  Parameter() = default;
220  Parameter(Parameter &&) = default;
221  Parameter(Parameter const &) = default;
222  Parameter &operator=(Parameter &&) = default;
223  Parameter &operator=(Parameter const &) = default;
224 
225  std::unique_ptr<AbstractParameter> to_heap() && override
226  {
227  return std::unique_ptr<AbstractParameter>(
228  new Parameter<Operation::CREATE_PATH>(std::move(*this)));
229  }
230 
231  std::string path = "";
232 };
233 
234 template <>
235 struct OPENPMDAPI_EXPORT Parameter<Operation::CLOSE_PATH>
236  : public AbstractParameter
237 {
238  Parameter() = default;
239  Parameter(Parameter &&) = default;
240  Parameter(Parameter const &) = default;
241  Parameter &operator=(Parameter &&) = default;
242  Parameter &operator=(Parameter const &) = default;
243 
244  std::unique_ptr<AbstractParameter> to_heap() && override
245  {
246  return std::unique_ptr<AbstractParameter>(
247  new Parameter<Operation::CLOSE_PATH>(std::move(*this)));
248  }
249 };
250 
251 template <>
252 struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_PATH>
253  : public AbstractParameter
254 {
255  Parameter() = default;
256  Parameter(Parameter &&) = default;
257  Parameter(Parameter const &) = default;
258  Parameter &operator=(Parameter &&) = default;
259  Parameter &operator=(Parameter const &) = default;
260 
261  std::unique_ptr<AbstractParameter> to_heap() && override
262  {
263  return std::unique_ptr<AbstractParameter>(
264  new Parameter<Operation::OPEN_PATH>(std::move(*this)));
265  }
266 
267  std::string path = "";
268 };
269 
270 template <>
271 struct OPENPMDAPI_EXPORT Parameter<Operation::DELETE_PATH>
272  : public AbstractParameter
273 {
274  Parameter() = default;
275  Parameter(Parameter &&) = default;
276  Parameter(Parameter const &) = default;
277  Parameter &operator=(Parameter &&) = default;
278  Parameter &operator=(Parameter const &) = default;
279 
280  std::unique_ptr<AbstractParameter> to_heap() && override
281  {
282  return std::unique_ptr<AbstractParameter>(
283  new Parameter<Operation::DELETE_PATH>(std::move(*this)));
284  }
285 
286  std::string path = "";
287 };
288 
289 template <>
290 struct OPENPMDAPI_EXPORT Parameter<Operation::LIST_PATHS>
291  : public AbstractParameter
292 {
293  Parameter() = default;
294  Parameter(Parameter &&) = default;
295  Parameter(Parameter const &) = default;
296  Parameter &operator=(Parameter &&) = default;
297  Parameter &operator=(Parameter const &) = default;
298 
299  std::unique_ptr<AbstractParameter> to_heap() && override
300  {
301  return std::unique_ptr<AbstractParameter>(
302  new Parameter<Operation::LIST_PATHS>(std::move(*this)));
303  }
304 
305  std::shared_ptr<std::vector<std::string>> paths =
306  std::make_shared<std::vector<std::string>>();
307 };
308 
309 template <>
310 struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
311  : public AbstractParameter
312 {
313  Parameter() = default;
314  Parameter(Parameter &&) = default;
315  Parameter(Parameter const &) = default;
316  Parameter &operator=(Parameter &&) = default;
317  Parameter &operator=(Parameter const &) = default;
318 
319  std::unique_ptr<AbstractParameter> to_heap() && override
320  {
321  return std::unique_ptr<AbstractParameter>(
322  new Parameter<Operation::CREATE_DATASET>(std::move(*this)));
323  }
324 
325  std::string name = "";
326  Extent extent = {};
327  Datatype dtype = Datatype::UNDEFINED;
328  std::string options = "{}";
329  std::optional<size_t> joinedDimension;
330 
337  template <typename TracingJSON>
338  static void warnUnusedParameters(
339  TracingJSON &,
340  std::string const &currentBackendName,
341  std::string const &warningMessage);
342 };
343 
344 template <>
345 struct OPENPMDAPI_EXPORT Parameter<Operation::EXTEND_DATASET>
346  : public AbstractParameter
347 {
348  Parameter() = default;
349  Parameter(Parameter &&) = default;
350  Parameter(Parameter const &) = default;
351  Parameter &operator=(Parameter &&) = default;
352  Parameter &operator=(Parameter const &) = default;
353 
354  std::unique_ptr<AbstractParameter> to_heap() && override
355  {
356  return std::unique_ptr<AbstractParameter>(
357  new Parameter<Operation::EXTEND_DATASET>(std::move(*this)));
358  }
359 
360  Extent extent = {};
361 };
362 
363 template <>
364 struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_DATASET>
365  : public AbstractParameter
366 {
367  Parameter() = default;
368  Parameter(Parameter &&) = default;
369  Parameter(Parameter const &) = default;
370  Parameter &operator=(Parameter &&) = default;
371  Parameter &operator=(Parameter const &) = default;
372 
373  std::unique_ptr<AbstractParameter> to_heap() && override
374  {
375  return std::unique_ptr<AbstractParameter>(
376  new Parameter<Operation::OPEN_DATASET>(std::move(*this)));
377  }
378 
379  std::string name = "";
380  std::shared_ptr<Datatype> dtype = std::make_shared<Datatype>();
381  std::shared_ptr<Extent> extent = std::make_shared<Extent>();
382 };
383 
384 template <>
385 struct OPENPMDAPI_EXPORT Parameter<Operation::DELETE_DATASET>
386  : public AbstractParameter
387 {
388  Parameter() = default;
389  Parameter(Parameter &&) = default;
390  Parameter(Parameter const &) = default;
391  Parameter &operator=(Parameter &&) = default;
392  Parameter &operator=(Parameter const &) = default;
393 
394  std::unique_ptr<AbstractParameter> to_heap() && override
395  {
396  return std::unique_ptr<AbstractParameter>(
397  new Parameter<Operation::DELETE_DATASET>(std::move(*this)));
398  }
399 
400  std::string name = "";
401 };
402 
403 template <>
404 struct OPENPMDAPI_EXPORT Parameter<Operation::WRITE_DATASET>
405  : public AbstractParameter
406 {
407  Parameter() = default;
408 
409  Parameter(Parameter &&) = default;
410  Parameter(Parameter const &) = delete;
411  Parameter &operator=(Parameter &&) = default;
412  Parameter &operator=(Parameter const &) = delete;
413 
414  std::unique_ptr<AbstractParameter> to_heap() && override
415  {
416  return std::unique_ptr<AbstractParameter>(
417  new Parameter<Operation::WRITE_DATASET>(std::move(*this)));
418  }
419 
420  Extent extent = {};
421  Offset offset = {};
422  Datatype dtype = Datatype::UNDEFINED;
424 };
425 
426 template <>
427 struct OPENPMDAPI_EXPORT Parameter<Operation::READ_DATASET>
428  : public AbstractParameter
429 {
430  Parameter() = default;
431  Parameter(Parameter &&) = default;
432  Parameter(Parameter const &) = default;
433  Parameter &operator=(Parameter &&) = default;
434  Parameter &operator=(Parameter const &) = default;
435 
436  std::unique_ptr<AbstractParameter> to_heap() && override
437  {
438  return std::unique_ptr<AbstractParameter>(
439  new Parameter<Operation::READ_DATASET>(std::move(*this)));
440  }
441 
442  Extent extent = {};
443  Offset offset = {};
444  Datatype dtype = Datatype::UNDEFINED;
445  std::shared_ptr<void> data = nullptr;
446 };
447 
448 template <>
449 struct OPENPMDAPI_EXPORT Parameter<Operation::LIST_DATASETS>
450  : public AbstractParameter
451 {
452  Parameter() = default;
453  Parameter(Parameter &&) = default;
454  Parameter(Parameter const &) = default;
455  Parameter &operator=(Parameter &&) = default;
456  Parameter &operator=(Parameter const &) = default;
457 
458  std::unique_ptr<AbstractParameter> to_heap() && override
459  {
460  return std::unique_ptr<AbstractParameter>(
461  new Parameter<Operation::LIST_DATASETS>(std::move(*this)));
462  }
463 
464  std::shared_ptr<std::vector<std::string>> datasets =
465  std::make_shared<std::vector<std::string>>();
466 };
467 
468 template <>
469 struct OPENPMDAPI_EXPORT Parameter<Operation::GET_BUFFER_VIEW>
470  : public AbstractParameter
471 {
472  Parameter() = default;
473  Parameter(Parameter &&) = default;
474  Parameter(Parameter const &) = default;
475  Parameter &operator=(Parameter &&) = default;
476  Parameter &operator=(Parameter const &) = default;
477 
478  std::unique_ptr<AbstractParameter> to_heap() && override
479  {
480  return std::unique_ptr<AbstractParameter>(
481  new Parameter<Operation::GET_BUFFER_VIEW>(std::move(*this)));
482  }
483 
484  // in parameters
485  Offset offset;
486  Extent extent;
487  Datatype dtype = Datatype::UNDEFINED;
488  bool update = false;
489  // out parameters
490  struct OutParameters
491  {
492  bool backendManagedBuffer = false;
493  unsigned viewIndex = 0;
494  void *ptr = nullptr;
495  };
496  std::shared_ptr<OutParameters> out = std::make_shared<OutParameters>();
497 };
498 
499 template <>
500 struct OPENPMDAPI_EXPORT Parameter<Operation::DELETE_ATT>
501  : public AbstractParameter
502 {
503  Parameter() = default;
504  Parameter(Parameter &&) = default;
505  Parameter(Parameter const &) = default;
506  Parameter &operator=(Parameter &&) = default;
507  Parameter &operator=(Parameter const &) = default;
508 
509  std::unique_ptr<AbstractParameter> to_heap() && override
510  {
511  return std::unique_ptr<AbstractParameter>(
512  new Parameter<Operation::DELETE_ATT>(std::move(*this)));
513  }
514 
515  std::string name = "";
516 };
517 
518 template <>
519 struct OPENPMDAPI_EXPORT Parameter<Operation::WRITE_ATT>
520  : public AbstractParameter
521 {
522  Parameter() = default;
523  Parameter(Parameter &&) = default;
524  Parameter(Parameter const &) = default;
525  Parameter &operator=(Parameter &&) = default;
526  Parameter &operator=(Parameter const &) = default;
527 
528  std::unique_ptr<AbstractParameter> to_heap() && override
529  {
530  return std::unique_ptr<AbstractParameter>(
531  new Parameter<Operation::WRITE_ATT>(std::move(*this)));
532  }
533 
534  std::string name = "";
535  Datatype dtype = Datatype::UNDEFINED;
536  /*
537  * If true, this attribute changes across IO steps.
538  * It should only be written in backends that support IO steps,
539  * otherwise writing should be skipped.
540  * The frontend is responsible for handling both situations.
541  */
542  enum class ChangesOverSteps
543  {
544  No,
545  Yes,
546  IfPossible
547  };
548  ChangesOverSteps changesOverSteps = ChangesOverSteps::No;
549  Attribute::resource resource;
550 };
551 
552 template <>
553 struct OPENPMDAPI_EXPORT Parameter<Operation::READ_ATT>
554  : public AbstractParameter
555 {
556  Parameter() = default;
557  Parameter(Parameter &&) = default;
558  Parameter(Parameter const &) = default;
559  Parameter &operator=(Parameter &&) = default;
560  Parameter &operator=(Parameter const &) = default;
561 
562  std::unique_ptr<AbstractParameter> to_heap() && override
563  {
564  return std::unique_ptr<AbstractParameter>(
565  new Parameter<Operation::READ_ATT>(std::move(*this)));
566  }
567 
568  std::string name = "";
569  std::shared_ptr<Datatype> dtype = std::make_shared<Datatype>();
570  std::shared_ptr<Attribute::resource> resource =
571  std::make_shared<Attribute::resource>();
572 };
573 
574 template <>
575 struct OPENPMDAPI_EXPORT Parameter<Operation::LIST_ATTS>
576  : public AbstractParameter
577 {
578  Parameter() = default;
579  Parameter(Parameter &&) = default;
580  Parameter(Parameter const &) = default;
581  Parameter &operator=(Parameter &&) = default;
582  Parameter &operator=(Parameter const &) = default;
583 
584  std::unique_ptr<AbstractParameter> to_heap() && override
585  {
586  return std::unique_ptr<AbstractParameter>(
587  new Parameter<Operation::LIST_ATTS>(std::move(*this)));
588  }
589 
590  std::shared_ptr<std::vector<std::string>> attributes =
591  std::make_shared<std::vector<std::string>>();
592 };
593 
594 template <>
595 struct OPENPMDAPI_EXPORT Parameter<Operation::ADVANCE>
596  : public AbstractParameter
597 {
598  Parameter() = default;
599  Parameter(Parameter &&) = default;
600  Parameter(Parameter const &) = default;
601  Parameter &operator=(Parameter &&) = default;
602  Parameter &operator=(Parameter const &) = default;
603 
604  std::unique_ptr<AbstractParameter> to_heap() && override
605  {
606  return std::unique_ptr<AbstractParameter>(
607  new Parameter<Operation::ADVANCE>(std::move(*this)));
608  }
609 
612  bool isThisStepMandatory = false;
614  std::shared_ptr<AdvanceStatus> status =
615  std::make_shared<AdvanceStatus>(AdvanceStatus::OK);
616 };
617 
618 template <>
619 struct OPENPMDAPI_EXPORT Parameter<Operation::AVAILABLE_CHUNKS>
620  : public AbstractParameter
621 {
622  Parameter() = default;
623  Parameter(Parameter &&) = default;
624  Parameter(Parameter const &) = default;
625  Parameter &operator=(Parameter &&) = default;
626  Parameter &operator=(Parameter const &) = default;
627 
628  std::unique_ptr<AbstractParameter> to_heap() && override
629  {
630  return std::unique_ptr<AbstractParameter>(
631  new Parameter<Operation::AVAILABLE_CHUNKS>(std::move(*this)));
632  }
633 
634  // output parameter
635  std::shared_ptr<ChunkTable> chunks = std::make_shared<ChunkTable>();
636 };
637 
638 template <>
639 struct OPENPMDAPI_EXPORT Parameter<Operation::DEREGISTER>
640  : public AbstractParameter
641 {
642  Parameter(void const *ptr_in) : former_parent(ptr_in)
643  {}
644 
645  Parameter(Parameter const &) = default;
646  Parameter(Parameter &&) = default;
647 
648  Parameter &operator=(Parameter const &) = default;
649  Parameter &operator=(Parameter &&) = default;
650 
651  std::unique_ptr<AbstractParameter> to_heap() && override
652  {
653  return std::make_unique<Parameter<Operation::DEREGISTER>>(
654  std::move(*this));
655  }
656 
657  // Just for verbose logging.
658  void const *former_parent = nullptr;
659 };
660 
669 class OPENPMDAPI_EXPORT IOTask
670 {
671 public:
680  template <Operation op>
681  explicit IOTask(Writable *w, Parameter<op> p)
682  : writable{w}, operation{op}, parameter{std::move(p).to_heap()}
683  {}
684 
685  template <Operation op>
686  explicit IOTask(Attributable *a, Parameter<op> p)
687  : writable{getWritable(a)}
688  , operation{op}
689  , parameter{std::move(p).to_heap()}
690  {}
691 
692  IOTask(IOTask const &other);
693  IOTask(IOTask &&other) noexcept;
694  IOTask &operator=(IOTask const &other);
695  IOTask &operator=(IOTask &&other) noexcept;
696 
697  Writable *writable;
698  Operation operation;
699  std::shared_ptr<AbstractParameter> parameter;
700 }; // IOTask
701 } // namespace openPMD
Layer to manage storage of attributes associated with file objects.
Definition: Attributable.hpp:101
Self-contained description of a single IO operation.
Definition: IOTask.hpp:670
IOTask(Writable *w, Parameter< op > p)
Constructor for self-contained description of single IO operation.
Definition: IOTask.hpp:681
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:75
Public definitions of openPMD-api.
Definition: Date.cpp:29
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:35
@ OK
stream goes on
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:51
OPENPMDAPI_EXPORT_ENUM_CLASS(Operation)
Type of IO operation between logical and persistent data.
Definition: IOTask.hpp:50
Definition: IOTask.hpp:78
AdvanceMode mode
input parameter
Definition: IOTask.hpp:611
static void warnUnusedParameters(TracingJSON &, std::string const &currentBackendName, std::string const &warningMessage)
Warn about unused JSON paramters.
Typesafe description of all required arguments for a specified Operation.
Definition: IOTask.hpp:105
Definition: Memory.hpp:175