openPMD-api
ADIOS2IOHandler.hpp
1 /* Copyright 2017-2021 Fabian Koller and Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
24 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
25 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp"
26 #include "openPMD/IO/AbstractIOHandler.hpp"
27 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
28 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
29 #include "openPMD/IO/FlushParametersInternal.hpp"
30 #include "openPMD/IO/IOTask.hpp"
31 #include "openPMD/IO/InvalidatableFile.hpp"
32 #include "openPMD/IterationEncoding.hpp"
33 #include "openPMD/auxiliary/JSON_internal.hpp"
34 #include "openPMD/backend/Writable.hpp"
35 #include "openPMD/config.hpp"
36 
37 #if openPMD_HAVE_ADIOS2
38 #include <adios2.h>
39 #endif
40 #if openPMD_HAVE_MPI
41 #include <mpi.h>
42 #endif
43 #include <nlohmann/json.hpp>
44 
45 #include <array>
46 #include <exception>
47 #include <future>
48 #include <iostream>
49 #include <memory> // shared_ptr
50 #include <optional>
51 #include <set>
52 #include <string>
53 #include <unordered_map>
54 #include <utility> // pair
55 #include <vector>
56 
57 namespace openPMD
58 {
59 #if openPMD_HAVE_ADIOS2
60 
61 class ADIOS2IOHandler;
62 
63 namespace detail
64 {
65  template <typename, typename>
66  struct DatasetHelper;
67  struct GetSpan;
68  struct DatasetReader;
69  struct AttributeReader;
70  struct AttributeWriter;
71  struct OldAttributeReader;
72  struct OldAttributeWriter;
73  template <typename>
75  struct DatasetOpener;
76  struct VariableDefiner;
77  template <typename>
78  struct DatasetTypes;
79  struct WriteDataset;
80  struct BufferedActions;
81  struct BufferedPut;
82  struct BufferedGet;
83  struct BufferedAttributeRead;
85  struct RunUniquePtrPut;
86 } // namespace detail
87 
88 namespace ADIOS2Schema
89 {
90  using schema_t = uint64_t;
91  /*
92  * Original ADIOS schema.
93  */
94  constexpr schema_t schema_0000_00_00 = 00000000;
95  /*
96  * This introduces attribute layout via scalar ADIOS variables.
97  */
98  constexpr schema_t schema_2021_02_09 = 20210209;
99 
100  enum class SupportedSchema : char
101  {
102  s_0000_00_00,
103  s_2021_02_09
104  };
105 } // namespace ADIOS2Schema
106 using SupportedSchema = ADIOS2Schema::SupportedSchema;
107 
109  : public AbstractIOHandlerImplCommon<ADIOS2FilePosition>
110 {
111  template <typename, typename>
112  friend struct detail::DatasetHelper;
113  friend struct detail::GetSpan;
114  friend struct detail::DatasetReader;
115  friend struct detail::AttributeReader;
116  friend struct detail::AttributeWriter;
117  friend struct detail::OldAttributeReader;
118  friend struct detail::OldAttributeWriter;
119  template <typename>
120  friend struct detail::AttributeTypes;
121  friend struct detail::DatasetOpener;
122  friend struct detail::VariableDefiner;
123  template <typename>
124  friend struct detail::DatasetTypes;
125  friend struct detail::WriteDataset;
126  friend struct detail::BufferedActions;
127  friend struct detail::BufferedAttributeRead;
128  friend struct detail::RunUniquePtrPut;
129 
130 public:
131 #if openPMD_HAVE_MPI
132 
135  MPI_Comm,
136  json::TracingJSON config,
137  std::string engineType,
138  std::string specifiedExtension);
139 
140 #endif // openPMD_HAVE_MPI
141 
142  explicit ADIOS2IOHandlerImpl(
144  json::TracingJSON config,
145  std::string engineType,
146  std::string specifiedExtension);
147 
148  ~ADIOS2IOHandlerImpl() override;
149 
150  std::future<void> flush(internal::ParsedFlushParams &);
151 
152  void
153  createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
154 
155  void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) override;
156 
157  // MPI Collective
158  bool checkFile(std::string fullFilePath) const;
159 
160  void
161  createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
162 
163  void createDataset(
164  Writable *, Parameter<Operation::CREATE_DATASET> const &) override;
165 
166  void extendDataset(
167  Writable *, Parameter<Operation::EXTEND_DATASET> const &) override;
168 
169  void openFile(Writable *, Parameter<Operation::OPEN_FILE> &) override;
170 
171  void
172  closeFile(Writable *, Parameter<Operation::CLOSE_FILE> const &) override;
173 
174  void openPath(Writable *, Parameter<Operation::OPEN_PATH> const &) override;
175 
176  void
177  closePath(Writable *, Parameter<Operation::CLOSE_PATH> const &) override;
178 
179  void openDataset(Writable *, Parameter<Operation::OPEN_DATASET> &) override;
180 
181  void
182  deleteFile(Writable *, Parameter<Operation::DELETE_FILE> const &) override;
183 
184  void
185  deletePath(Writable *, Parameter<Operation::DELETE_PATH> const &) override;
186 
187  void deleteDataset(
188  Writable *, Parameter<Operation::DELETE_DATASET> const &) override;
189 
190  void deleteAttribute(
191  Writable *, Parameter<Operation::DELETE_ATT> const &) override;
192 
193  void
194  writeDataset(Writable *, Parameter<Operation::WRITE_DATASET> &) override;
195 
196  void writeAttribute(
197  Writable *, Parameter<Operation::WRITE_ATT> const &) override;
198 
199  void readDataset(Writable *, Parameter<Operation::READ_DATASET> &) override;
200 
201  void
202  getBufferView(Writable *, Parameter<Operation::GET_BUFFER_VIEW> &) override;
203 
204  void readAttribute(Writable *, Parameter<Operation::READ_ATT> &) override;
205 
206  void listPaths(Writable *, Parameter<Operation::LIST_PATHS> &) override;
207 
208  void
209  listDatasets(Writable *, Parameter<Operation::LIST_DATASETS> &) override;
210 
211  void listAttributes(
212  Writable *, Parameter<Operation::LIST_ATTS> &parameters) override;
213 
214  void advance(Writable *, Parameter<Operation::ADVANCE> &) override;
215 
216  void availableChunks(
218 
219  void
220  deregister(Writable *, Parameter<Operation::DEREGISTER> const &) override;
221 
226  adios2::Mode adios2AccessMode(std::string const &fullPath);
227 
228  enum class FlushTarget : unsigned char
229  {
230  Buffer,
231  Buffer_Override,
232  Disk,
233  Disk_Override
234  };
235 
236  FlushTarget m_flushTarget = FlushTarget::Disk;
237 
238 private:
239  adios2::ADIOS m_ADIOS;
240 #if openPMD_HAVE_MPI
241  std::optional<MPI_Comm> m_communicator;
242 #endif
243  /*
244  * If the iteration encoding is variableBased, we default to using the
245  * 2021_02_09 schema since it allows mutable attributes.
246  */
247  IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
251  std::string m_engineType;
252  /*
253  * The filename extension specified by the user.
254  */
255  std::string m_userSpecifiedExtension;
256 
257  /*
258  * Empty option: No schema has been explicitly selected, use default.
259  */
260  std::optional<ADIOS2Schema::schema_t> m_schema;
261 
262  enum class UseSpan : char
263  {
264  Yes,
265  No,
266  Auto
267  };
268 
269  UseSpan m_useSpanBasedPutByDefault = UseSpan::Auto;
270 
271  enum class AttributeLayout : char
272  {
273  ByAdiosAttributes,
274  ByAdiosVariables
275  };
276 
277  inline SupportedSchema schema() const
278  {
279  if (!m_schema.has_value())
280  {
281  return SupportedSchema::s_0000_00_00;
282  }
283  switch (m_schema.value())
284  {
285  case ADIOS2Schema::schema_0000_00_00:
286  return SupportedSchema::s_0000_00_00;
287  case ADIOS2Schema::schema_2021_02_09:
288  return SupportedSchema::s_2021_02_09;
289  default:
290  throw std::runtime_error(
291  "[ADIOS2] Encountered unsupported schema version: " +
292  std::to_string(m_schema.value()));
293  }
294  }
295 
296  inline AttributeLayout attributeLayout() const
297  {
298  switch (schema())
299  {
300  case SupportedSchema::s_0000_00_00:
301  return AttributeLayout::ByAdiosAttributes;
302  case SupportedSchema::s_2021_02_09:
303  return AttributeLayout::ByAdiosVariables;
304  }
305  throw std::runtime_error("Unreachable!");
306  }
307 
308  struct ParameterizedOperator
309  {
310  adios2::Operator op;
311  adios2::Params params;
312  };
313 
314  std::vector<ParameterizedOperator> defaultOperators;
315 
316  json::TracingJSON m_config;
317  static json::TracingJSON nullvalue;
318 
319  void init(json::TracingJSON config);
320 
321  template <typename Key>
322  json::TracingJSON config(Key &&key, json::TracingJSON &cfg)
323  {
324  if (cfg.json().is_object() && cfg.json().contains(key))
325  {
326  return cfg[key];
327  }
328  else
329  {
330  return nullvalue;
331  }
332  }
333 
334  template <typename Key>
335  json::TracingJSON config(Key &&key)
336  {
337  return config<Key>(std::forward<Key>(key), m_config);
338  }
339 
347  std::optional<std::vector<ParameterizedOperator>>
348  getOperators(json::TracingJSON config);
349 
350  // use m_config
351  std::optional<std::vector<ParameterizedOperator>> getOperators();
352 
353  std::string fileSuffix(bool verbose = true) const;
354 
355  /*
356  * We need to give names to IO objects. These names are irrelevant
357  * within this application, since:
358  * 1) The name of the file written to is decided by the opened Engine's
359  * name.
360  * 2) The IOs are managed by the unordered_map m_fileData, so we do not
361  * need the ADIOS2 internal management.
362  * Since within one m_ADIOS object, the same IO name cannot be used more
363  * than once, we ensure different names by using the name counter.
364  * This allows to overwrite a file later without error.
365  */
366  int nameCounter{0};
367 
368  /*
369  * IO-heavy actions are deferred to a later point. This map stores for
370  * each open file (identified by an InvalidatableFile object) an object
371  * that manages IO-heavy actions, as well as its ADIOS2 objects, i.e.
372  * IO and Engine object.
373  * Not to be accessed directly, use getFileData().
374  */
375  std::unordered_map<
377  std::unique_ptr<detail::BufferedActions>>
378  m_fileData;
379 
380  std::map<std::string, adios2::Operator> m_operators;
381 
382  // Overrides from AbstractIOHandlerImplCommon.
383 
384  std::string
385  filePositionToString(std::shared_ptr<ADIOS2FilePosition>) override;
386 
387  std::shared_ptr<ADIOS2FilePosition> extendFilePosition(
388  std::shared_ptr<ADIOS2FilePosition> const &pos,
389  std::string extend) override;
390 
391  // Helper methods.
392 
393  std::optional<adios2::Operator>
394  getCompressionOperator(std::string const &compression);
395 
396  /*
397  * The name of the ADIOS2 variable associated with this Writable.
398  * To be used for Writables that represent a dataset.
399  */
400  std::string nameOfVariable(Writable *writable);
401 
411  std::string nameOfAttribute(Writable *writable, std::string attribute);
412 
413  /*
414  * Figure out whether the Writable corresponds with a
415  * group or a dataset.
416  */
417  ADIOS2FilePosition::GD groupOrDataset(Writable *);
418 
419  enum class IfFileNotOpen : bool
420  {
421  OpenImplicitly,
422  ThrowError
423  };
424 
425  detail::BufferedActions &getFileData(InvalidatableFile file, IfFileNotOpen);
426 
427  void dropFileData(InvalidatableFile file);
428 
429  /*
430  * Prepare a variable that already exists for an IO
431  * operation, including:
432  * (1) checking that its datatype matches T.
433  * (2) the offset and extent match the variable's shape
434  * (3) setting the offset and extent (ADIOS lingo: start
435  * and count)
436  */
437  template <typename T>
438  adios2::Variable<T> verifyDataset(
439  Offset const &offset,
440  Extent const &extent,
441  adios2::IO &IO,
442  std::string const &var);
443 }; // ADIOS2IOHandlerImpl
444 
445 /*
446  * The following strings are used during parsing of the JSON configuration
447  * string for the ADIOS2 backend.
448  */
449 namespace ADIOS2Defaults
450 {
451  using const_str = char const *const;
452  constexpr const_str str_engine = "engine";
453  constexpr const_str str_type = "type";
454  constexpr const_str str_params = "parameters";
455  constexpr const_str str_usesteps = "usesteps";
456  constexpr const_str str_flushtarget = "preferred_flush_target";
457  constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps";
458  constexpr const_str str_adios2Schema =
459  "__openPMD_internal/openPMD2_adios2_schema";
460  constexpr const_str str_isBooleanOldLayout = "__is_boolean__";
461  constexpr const_str str_isBooleanNewLayout =
462  "__openPMD_internal/is_boolean";
463 } // namespace ADIOS2Defaults
464 
465 namespace detail
466 {
467  // Helper structs for calls to the switchType function
468 
469  template <typename T>
470  inline constexpr bool IsUnsupportedComplex_v =
471  std::is_same_v<T, std::complex<long double>> ||
472  std::is_same_v<T, std::vector<std::complex<long double>>>;
473 
475  {
476  template <typename T>
477  static void call(
478  ADIOS2IOHandlerImpl *impl,
479  BufferedGet &bp,
480  adios2::IO &IO,
481  adios2::Engine &engine,
482  std::string const &fileName);
483 
484  static constexpr char const *errorMsg = "ADIOS2: readDataset()";
485  };
486 
488  {
489  template <typename T>
490  static Datatype call(
491  adios2::IO &IO,
492  std::string name,
493  std::shared_ptr<Attribute::resource> resource);
494 
495  template <int n, typename... Params>
496  static Datatype call(Params &&...);
497  };
498 
500  {
501  template <typename T>
502  static void call(
503  ADIOS2IOHandlerImpl *impl,
504  Writable *writable,
505  const Parameter<Operation::WRITE_ATT> &parameters);
506 
507  template <int n, typename... Params>
508  static void call(Params &&...);
509  };
510 
512  {
513  template <typename T>
514  static Datatype call(
515  adios2::IO &IO,
516  detail::PreloadAdiosAttributes const &preloadedAttributes,
517  std::string name,
518  std::shared_ptr<Attribute::resource> resource);
519 
520  template <int n, typename... Params>
521  static Datatype call(Params &&...);
522  };
523 
525  {
526  template <typename T>
527  static void
528  call(detail::BufferedAttributeWrite &params, BufferedActions &fileData);
529 
530  template <int n, typename... Params>
531  static void call(Params &&...);
532  };
533 
535  {
536  template <typename T>
537  static void call(
538  ADIOS2IOHandlerImpl *impl,
540  const std::string &varName,
542 
543  static constexpr char const *errorMsg = "ADIOS2: openDataset()";
544  };
545 
547  {
548  template <typename T>
549  static void call(BufferedActions &ba, BufferedPut &bp);
550 
551  template <int n, typename... Params>
552  static void call(Params &&...);
553  };
554 
556  {
572  template <typename T>
573  static void call(
574  adios2::IO &IO,
575  std::string const &name,
576  std::vector<ADIOS2IOHandlerImpl::ParameterizedOperator> const
577  &compressions,
578  adios2::Dims const &shape = adios2::Dims(),
579  adios2::Dims const &start = adios2::Dims(),
580  adios2::Dims const &count = adios2::Dims(),
581  bool const constantDims = false);
582 
583  static constexpr char const *errorMsg = "ADIOS2: defineVariable()";
584  };
585 
587  {
588  template <typename T>
589  static void call(
591  adios2::IO &IO,
592  adios2::Engine &engine,
593  std::string const &varName,
594  bool allSteps);
595 
596  template <int n, typename... Params>
597  static void call(Params &&...);
598  };
599 
600  // Helper structs to help distinguish valid attribute/variable
601  // datatypes from invalid ones
602 
603  /*
604  * This struct's purpose is to have specialisations
605  * for vector and array types, as well as the boolean
606  * type (which is not natively supported by ADIOS).
607  */
608  template <typename T>
609  struct AttributeTypes
610  {
611  static void createAttribute(
612  adios2::IO &IO,
613  adios2::Engine &engine,
615  T value);
616 
617  static Datatype readAttribute(
619  std::string name,
620  std::shared_ptr<Attribute::resource> resource);
621 
626  static bool attributeUnchanged(adios2::IO &IO, std::string name, T val)
627  {
628  auto attr = IO.InquireAttribute<T>(name);
629  if (!attr)
630  {
631  return false;
632  }
633  std::vector<T> data = attr.Data();
634  if (data.size() != 1)
635  {
636  return false;
637  }
638  return data[0] == val;
639  }
640  };
641 
642  template <>
643  struct AttributeTypes<std::complex<long double>>
644  {
645  static void createAttribute(
646  adios2::IO &,
647  adios2::Engine &,
649  std::complex<long double>)
650  {
651  throw std::runtime_error(
652  "[ADIOS2] Internal error: no support for long double complex "
653  "attribute types");
654  }
655 
656  static Datatype readAttribute(
658  std::string,
659  std::shared_ptr<Attribute::resource>)
660  {
661  throw std::runtime_error(
662  "[ADIOS2] Internal error: no support for long double complex "
663  "attribute types");
664  }
665 
666  static bool
667  attributeUnchanged(adios2::IO &, std::string, std::complex<long double>)
668  {
669  throw std::runtime_error(
670  "[ADIOS2] Internal error: no support for long double complex "
671  "attribute types");
672  }
673  };
674 
675  template <>
676  struct AttributeTypes<std::vector<std::complex<long double>>>
677  {
678  static void createAttribute(
679  adios2::IO &,
680  adios2::Engine &,
682  const std::vector<std::complex<long double>> &)
683  {
684  throw std::runtime_error(
685  "[ADIOS2] Internal error: no support for long double complex "
686  "vector attribute types");
687  }
688 
689  static Datatype readAttribute(
691  std::string,
692  std::shared_ptr<Attribute::resource>)
693  {
694  throw std::runtime_error(
695  "[ADIOS2] Internal error: no support for long double complex "
696  "vector attribute types");
697  }
698 
699  static bool attributeUnchanged(
700  adios2::IO &, std::string, std::vector<std::complex<long double>>)
701  {
702  throw std::runtime_error(
703  "[ADIOS2] Internal error: no support for long double complex "
704  "vector attribute types");
705  }
706  };
707 
708  template <typename T>
709  struct AttributeTypes<std::vector<T>>
710  {
711  static void createAttribute(
712  adios2::IO &IO,
713  adios2::Engine &engine,
715  const std::vector<T> &value);
716 
717  static Datatype readAttribute(
719  std::string name,
720  std::shared_ptr<Attribute::resource> resource);
721 
722  static bool
723  attributeUnchanged(adios2::IO &IO, std::string name, std::vector<T> val)
724  {
725  auto attr = IO.InquireAttribute<T>(name);
726  if (!attr)
727  {
728  return false;
729  }
730  std::vector<T> data = attr.Data();
731  if (data.size() != val.size())
732  {
733  return false;
734  }
735  for (size_t i = 0; i < val.size(); ++i)
736  {
737  if (data[i] != val[i])
738  {
739  return false;
740  }
741  }
742  return true;
743  }
744  };
745 
746  template <>
747  struct AttributeTypes<std::vector<std::string>>
748  {
749  static void createAttribute(
750  adios2::IO &IO,
751  adios2::Engine &engine,
753  const std::vector<std::string> &vec);
754 
755  static Datatype readAttribute(
757  std::string name,
758  std::shared_ptr<Attribute::resource> resource);
759 
760  static bool attributeUnchanged(
761  adios2::IO &IO, std::string name, std::vector<std::string> val)
762  {
763  auto attr = IO.InquireAttribute<std::string>(name);
764  if (!attr)
765  {
766  return false;
767  }
768  std::vector<std::string> data = attr.Data();
769  if (data.size() != val.size())
770  {
771  return false;
772  }
773  for (size_t i = 0; i < val.size(); ++i)
774  {
775  if (data[i] != val[i])
776  {
777  return false;
778  }
779  }
780  return true;
781  }
782  };
783 
784  template <typename T, size_t n>
785  struct AttributeTypes<std::array<T, n>>
786  {
787  static void createAttribute(
788  adios2::IO &IO,
789  adios2::Engine &engine,
791  const std::array<T, n> &value);
792 
793  static Datatype readAttribute(
795  std::string name,
796  std::shared_ptr<Attribute::resource> resource);
797 
798  static bool attributeUnchanged(
799  adios2::IO &IO, std::string name, std::array<T, n> val)
800  {
801  auto attr = IO.InquireAttribute<T>(name);
802  if (!attr)
803  {
804  return false;
805  }
806  std::vector<T> data = attr.Data();
807  if (data.size() != n)
808  {
809  return false;
810  }
811  for (size_t i = 0; i < n; ++i)
812  {
813  if (data[i] != val[i])
814  {
815  return false;
816  }
817  }
818  return true;
819  }
820  };
821 
822  namespace bool_repr
823  {
824  using rep = detail::bool_representation;
825 
826  static constexpr rep toRep(bool b)
827  {
828  return b ? 1U : 0U;
829  }
830 
831  static constexpr bool fromRep(rep r)
832  {
833  return r != 0;
834  }
835  } // namespace bool_repr
836 
837  template <>
838  struct AttributeTypes<bool>
839  {
840  using rep = detail::bool_representation;
841 
842  static constexpr rep toRep(bool b)
843  {
844  return b ? 1U : 0U;
845  }
846 
847  static constexpr bool fromRep(rep r)
848  {
849  return r != 0;
850  }
851 
852  static void createAttribute(
853  adios2::IO &IO,
854  adios2::Engine &engine,
856  bool value);
857 
858  static Datatype readAttribute(
860  std::string name,
861  std::shared_ptr<Attribute::resource> resource);
862 
863  static bool
864  attributeUnchanged(adios2::IO &IO, std::string name, bool val)
865  {
866  auto attr = IO.InquireAttribute<rep>(name);
867  if (!attr)
868  {
869  return false;
870  }
871  std::vector<rep> data = attr.Data();
872  if (data.size() != 1)
873  {
874  return false;
875  }
876  return data[0] == toRep(val);
877  }
878  };
879 
880  // Other datatypes used in the ADIOS2IOHandler implementation
881 
882  struct BufferedActions;
883 
884  /*
885  * IO-heavy action to be executed upon flushing.
886  */
888  {
889  explicit BufferedAction() = default;
890  virtual ~BufferedAction() = default;
891 
892  BufferedAction(BufferedAction const &other) = delete;
893  BufferedAction(BufferedAction &&other) = default;
894 
895  BufferedAction &operator=(BufferedAction const &other) = delete;
896  BufferedAction &operator=(BufferedAction &&other) = default;
897 
898  virtual void run(BufferedActions &) = 0;
899  };
900 
902  {
903  std::string name;
905 
906  void run(BufferedActions &) override;
907  };
908 
910  {
911  std::string name;
913 
914  void run(BufferedActions &) override;
915  };
916 
918  {
919  std::string name;
920  Offset offset;
921  Extent extent;
923  Datatype dtype;
924 
925  void run(BufferedActions &);
926  };
927 
929  {
931  std::string name;
932 
933  void run(BufferedActions &) override;
934  };
935 
937  {
939  std::string name;
940 
941  void run(BufferedActions &);
942  };
943 
945  {
946  std::string name;
947  Datatype dtype;
948  Attribute::resource resource;
949  std::vector<char> bufferForVecString;
950 
951  void run(BufferedActions &) override;
952  };
953 
955  {
956  virtual void *update() = 0;
957  virtual ~I_UpdateSpan() = default;
958  };
959 
960  template <typename T>
962  {
963  adios2::detail::Span<T> span;
964 
965  UpdateSpan(adios2::detail::Span<T>);
966 
967  void *update() override;
968  };
969 
970  /*
971  * Manages per-file information about
972  * (1) the file's IO and Engine objects
973  * (2) the file's deferred IO-heavy actions
974  */
976  {
977  friend struct BufferedGet;
978  friend struct BufferedPut;
979  friend struct RunUniquePtrPut;
980  friend struct WriteDataset;
981 
982  using FlushTarget = ADIOS2IOHandlerImpl::FlushTarget;
983 
984  BufferedActions(BufferedActions const &) = delete;
985 
993  std::string m_file;
1012  std::string m_IOName;
1013  adios2::ADIOS &m_ADIOS;
1014  adios2::IO m_IO;
1019  std::vector<std::unique_ptr<BufferedAction>> m_buffer;
1030  std::map<std::string, BufferedAttributeWrite> m_attributeWrites;
1036  std::vector<BufferedAttributeRead> m_attributeReads;
1043  std::vector<BufferedUniquePtrPut> m_uniquePtrPuts;
1050  std::vector<std::unique_ptr<BufferedAction>> m_alreadyEnqueued;
1051  adios2::Mode m_mode;
1060  std::map<unsigned, std::unique_ptr<I_UpdateSpan>> m_updateSpans;
1061  PreloadAdiosAttributes preloadAttributes;
1062 
1063  /*
1064  * We call an attribute committed if the step during which it was
1065  * written has been closed.
1066  * A committed attribute cannot be modified.
1067  */
1068  std::set<std::string> uncommittedAttributes;
1069 
1070  /*
1071  * The openPMD API will generally create new attributes for each
1072  * iteration. This results in a growing number of attributes over time.
1073  * In streaming-based modes, these will be completely sent anew in each
1074  * iteration. If the following boolean is true, old attributes will be
1075  * removed upon CLOSE_GROUP.
1076  * Should not be set to true in persistent backends.
1077  * Will be automatically set by BufferedActions::configure_IO depending
1078  * on chosen ADIOS2 engine and can not be explicitly overridden by user.
1079  */
1080  bool optimizeAttributesStreaming = false;
1081 
1082  using ParsePreference =
1084  ParsePreference parsePreference = ParsePreference::UpFront;
1085 
1086  using AttributeMap_t = std::map<std::string, adios2::Params>;
1087 
1089 
1090  ~BufferedActions();
1091 
1096  void finalize();
1097 
1098  adios2::Engine &getEngine();
1099  adios2::Engine &requireActiveStep();
1100 
1101  template <typename BA>
1102  void enqueue(BA &&ba);
1103 
1104  template <typename BA>
1105  void enqueue(BA &&ba, decltype(m_buffer) &);
1106 
1107  template <typename... Args>
1108  void flush(Args &&...args);
1109 
1111  {
1112  /*
1113  * Only execute performPutsGets if UserFlush.
1114  */
1115  FlushLevel level;
1116  FlushTarget flushTarget = FlushTarget::Disk;
1117 
1118  ADIOS2FlushParams(FlushLevel level_in) : level(level_in)
1119  {}
1120 
1121  ADIOS2FlushParams(FlushLevel level_in, FlushTarget flushTarget_in)
1122  : level(level_in), flushTarget(flushTarget_in)
1123  {}
1124  };
1125 
1144  template <typename F>
1145  void flush_impl(
1146  ADIOS2FlushParams flushParams,
1147  F &&performPutsGets,
1148  bool writeLatePuts,
1149  bool flushUnconditionally);
1150 
1156  void flush_impl(ADIOS2FlushParams, bool writeLatePuts = false);
1157 
1170  AdvanceStatus advance(AdvanceMode mode, bool calledExplicitly);
1171 
1172  /*
1173  * Delete all buffered actions without running them.
1174  */
1175  void drop();
1176 
1177  AttributeMap_t const &availableAttributes();
1178 
1179  std::vector<std::string>
1180  availableAttributesPrefixed(std::string const &prefix);
1181 
1182  /*
1183  * See description below.
1184  */
1185  void invalidateAttributesMap();
1186 
1187  AttributeMap_t const &availableVariables();
1188 
1189  std::vector<std::string>
1190  availableVariablesPrefixed(std::string const &prefix);
1191 
1192  /*
1193  * See description below.
1194  */
1195  void invalidateVariablesMap();
1196 
1197  /*
1198  * streamStatus is NoStream for file-based ADIOS engines.
1199  * This is relevant for the method BufferedActions::requireActiveStep,
1200  * where a step is only opened if the status is OutsideOfStep, but not
1201  * if NoStream. The rationale behind this is that parsing a Series
1202  * works differently for file-based and for stream-based engines:
1203  * * stream-based: Iterations are parsed as they arrive. For parsing an
1204  * iteration, the iteration must be awaited.
1205  * BufferedActions::requireActiveStep takes care of this.
1206  * * file-based: The Series is parsed up front. If no step has been
1207  * opened yet, ADIOS2 gives access to all variables and attributes
1208  * from all steps. Upon opening a step, only the variables from that
1209  * step are shown which hinders parsing. So, until a step is
1210  * explicitly opened via ADIOS2IOHandlerImpl::advance, do not open
1211  * one.
1212  * This is to enable use of ADIOS files without the Streaming API
1213  * (i.e. all iterations should be visible to the user upon opening
1214  * the Series.)
1215  * @todo Add a workflow without up-front parsing of all iterations
1216  * for file-based engines.
1217  * (This would merely be an optimization since the streaming
1218  * API still works with files as intended.)
1219  *
1220  */
1221  enum class StreamStatus
1222  {
1226  DuringStep,
1230  OutsideOfStep,
1234  StreamOver,
1251  NoStream,
1278  Parsing,
1284  Undecided
1285  };
1286  StreamStatus streamStatus = StreamStatus::OutsideOfStep;
1287 
1288  private:
1289  ADIOS2IOHandlerImpl *m_impl;
1290  std::optional<adios2::Engine> m_engine;
1291 
1294  std::string m_engineType;
1295 
1296  /*
1297  * ADIOS2 does not give direct access to its internal attribute and
1298  * variable maps, but will instead give access to copies of them.
1299  * In order to avoid unnecessary copies, we buffer the returned map.
1300  * The downside of this is that we need to pay attention to invalidate
1301  * the map whenever an attribute/variable is altered. In that case, we
1302  * fetch the map anew.
1303  * If empty, the buffered map has been invalidated and needs to be
1304  * queried from ADIOS2 again. If full, the buffered map is equivalent to
1305  * the map that would be returned by a call to
1306  * IO::Available(Attributes|Variables).
1307  */
1308  std::optional<AttributeMap_t> m_availableAttributes;
1309  std::optional<AttributeMap_t> m_availableVariables;
1310 
1311  /*
1312  * Cannot write attributes right after opening the engine
1313  * https://github.com/ornladios/ADIOS2/issues/3433
1314  */
1315  bool initializedDefaults = false;
1316  /*
1317  * finalize() will set this true to avoid running twice.
1318  */
1319  bool finalized = false;
1320 
1321  inline SupportedSchema schema() const
1322  {
1323  return m_impl->schema();
1324  }
1325 
1326  void create_IO();
1327 
1328  void configure_IO(ADIOS2IOHandlerImpl &impl);
1329  void configure_IO_Read(std::optional<bool> userSpecifiedUsesteps);
1330  void configure_IO_Write(std::optional<bool> userSpecifiedUsesteps);
1331 
1332  using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout;
1333  inline AttributeLayout attributeLayout() const
1334  {
1335  return m_impl->attributeLayout();
1336  }
1337  };
1338 
1339 } // namespace detail
1340 #endif // openPMD_HAVE_ADIOS2
1341 
1343 {
1344 #if openPMD_HAVE_ADIOS2
1345 
1346  friend class ADIOS2IOHandlerImpl;
1347 
1348 private:
1349  ADIOS2IOHandlerImpl m_impl;
1350 
1351 public:
1352  ~ADIOS2IOHandler() override
1353  {
1354  // we must not throw in a destructor
1355  try
1356  {
1357  auto params = internal::defaultParsedFlushParams;
1358  this->flush(params);
1359  }
1360  catch (std::exception const &ex)
1361  {
1362  std::cerr << "[~ADIOS2IOHandler] An error occurred: " << ex.what()
1363  << std::endl;
1364  }
1365  catch (...)
1366  {
1367  std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
1368  }
1369  }
1370 
1371 #else
1372 public:
1373 #endif
1374 
1375 #if openPMD_HAVE_MPI
1376 
1378  std::string path,
1379  Access,
1380  MPI_Comm,
1381  json::TracingJSON options,
1382  std::string engineType,
1383  std::string specifiedExtension);
1384 
1385 #endif
1386 
1388  std::string path,
1389  Access,
1390  json::TracingJSON options,
1391  std::string engineType,
1392  std::string specifiedExtension);
1393 
1394  std::string backendName() const override
1395  {
1396  return "ADIOS2";
1397  }
1398 
1399  std::future<void> flush(internal::ParsedFlushParams &) override;
1400 }; // ADIOS2IOHandler
1401 } // namespace openPMD
Definition: ADIOS2IOHandler.hpp:66
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON_internal.hpp:67
Wrapper around a shared pointer to:
Definition: InvalidatableFile.hpp:43
Definition: ADIOS2IOHandler.hpp:586
FlushLevel
Determine what items should be flushed upon Series::flush()
Definition: AbstractIOHandler.hpp:47
Definition: ADIOS2IOHandler.hpp:954
Definition: ADIOS2IOHandler.hpp:936
Definition: ADIOS2IOHandler.hpp:499
nlohmann::json & json()
Access the underlying JSON value.
Definition: JSON_internal.hpp:79
Definition: ADIOS2IOHandler.hpp:511
StreamStatus
Definition: ADIOS2IOHandler.hpp:1221
Access
File access mode to use during IO.
Definition: Access.hpp:29
Definition: ADIOS2IOHandler.hpp:917
Definition: ADIOS2IOHandler.hpp:555
STL namespace.
std::string m_file
The full path to the file created on disk, including the containing directory and the file extension...
Definition: ADIOS2IOHandler.hpp:993
Definition: ADIOS2IOHandler.hpp:108
Definition: ADIOS2IOHandler.hpp:975
std::vector< std::unique_ptr< BufferedAction > > m_buffer
The default queue for deferred actions.
Definition: ADIOS2IOHandler.hpp:1019
Definition: ADIOS2IOHandler.hpp:909
Definition: ADIOS2IOHandler.cpp:898
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
Definition: ADIOS2IOHandler.hpp:78
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:34
Definition: ADIOS2IOHandler.hpp:928
Definition: ADIOS2IOHandler.hpp:901
Definition: ADIOS2IOHandler.hpp:944
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:45
Definition: Container.cpp:51
std::vector< BufferedUniquePtrPut > m_uniquePtrPuts
When receiving a unique_ptr, we know that the buffer is ours and ours alone.
Definition: ADIOS2IOHandler.hpp:1043
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:179
Definition: ADIOS2IOHandler.hpp:1110
std::map< unsigned, std::unique_ptr< I_UpdateSpan > > m_updateSpans
The base pointer of an ADIOS2 span might change after reallocations.
Definition: ADIOS2IOHandler.hpp:1060
Public definitions of openPMD-api.
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:64
Definition: ADIOS2IOHandler.hpp:1342
Definition: ADIOS2IOHandler.hpp:546
std::string backendName() const override
The currently used backend.
Definition: ADIOS2IOHandler.hpp:1394
Typesafe description of all required arguments for a specified Operation.
Definition: IOTask.hpp:120
Class that is responsible for scheduling and buffering openPMD attribute loads from ADIOS2...
Definition: ADIOS2PreloadAttributes.hpp:64
Definition: ADIOS2IOHandler.hpp:524
std::vector< BufferedAttributeRead > m_attributeReads
Definition: ADIOS2IOHandler.hpp:1036
IterationEncoding
Encoding scheme of an Iterations Series&#39;.
Definition: IterationEncoding.hpp:32
Definition: Writable.hpp:44
static bool attributeUnchanged(adios2::IO &IO, std::string name, T val)
Is the attribute given by parameters name and val already defined exactly in that way within the give...
Definition: ADIOS2IOHandler.hpp:626
Definition: ADIOS2IOHandler.hpp:534
std::map< std::string, BufferedAttributeWrite > m_attributeWrites
Buffer for attributes to be written in the new (variable-based) attribute layout. ...
Definition: ADIOS2IOHandler.hpp:1030
Definition: ADIOS2IOHandler.hpp:474
Definition: ADIOS2IOHandler.cpp:2430
std::string m_IOName
ADIOS requires giving names to instances of adios2::IO.
Definition: ADIOS2IOHandler.hpp:1012
Definition: ADIOS2IOHandler.hpp:961
Definition: ADIOS2IOHandler.hpp:74
Definition: ADIOS2IOHandler.hpp:887
Definition: ADIOS2IOHandler.hpp:487
Definition: FlushParametersInternal.hpp:31
std::vector< std::unique_ptr< BufferedAction > > m_alreadyEnqueued
This contains deferred actions that have already been enqueued into ADIOS2, but not yet performed in ...
Definition: ADIOS2IOHandler.hpp:1050