23 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" 24 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" 25 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" 26 #include "openPMD/IO/AbstractIOHandler.hpp" 27 #include "openPMD/IO/AbstractIOHandlerImpl.hpp" 28 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" 29 #include "openPMD/IO/IOTask.hpp" 30 #include "openPMD/IO/InvalidatableFile.hpp" 31 #include "openPMD/IterationEncoding.hpp" 32 #include "openPMD/auxiliary/JSON.hpp" 33 #include "openPMD/auxiliary/Option.hpp" 34 #include "openPMD/backend/Writable.hpp" 35 #include "openPMD/config.hpp" 37 #if openPMD_HAVE_ADIOS2 43 #include <nlohmann/json.hpp> 52 #include <unordered_map> 58 #if openPMD_HAVE_ADIOS2 60 class ADIOS2IOHandler;
64 template <
typename,
typename>
86 namespace ADIOS2Schema
88 using schema_t = uint64_t;
92 constexpr schema_t schema_0000_00_00 = 00000000;
96 constexpr schema_t schema_2021_02_09 = 20210209;
98 enum class SupportedSchema : char
104 using SupportedSchema = ADIOS2Schema::SupportedSchema;
109 template <
typename,
typename>
133 nlohmann::json config,
134 std::string engineType);
136 #endif // openPMD_HAVE_MPI 178 void deleteAttribute(
204 void availableChunks(
210 adios2::Mode adios2AccessMode(std::string
const &fullPath);
213 adios2::ADIOS m_ADIOS;
222 std::string m_engineType;
223 ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00;
225 enum class UseSpan : char
232 UseSpan m_useSpanBasedPutByDefault = UseSpan::Auto;
234 enum class AttributeLayout : char
240 inline SupportedSchema schema()
const 244 case ADIOS2Schema::schema_0000_00_00:
245 return SupportedSchema::s_0000_00_00;
246 case ADIOS2Schema::schema_2021_02_09:
247 return SupportedSchema::s_2021_02_09;
249 throw std::runtime_error(
250 "[ADIOS2] Encountered unsupported schema version: " +
251 std::to_string(m_schema));
255 inline AttributeLayout attributeLayout()
const 259 case SupportedSchema::s_0000_00_00:
260 return AttributeLayout::ByAdiosAttributes;
261 case SupportedSchema::s_2021_02_09:
262 return AttributeLayout::ByAdiosVariables;
264 throw std::runtime_error(
"Unreachable!");
267 struct ParameterizedOperator
270 adios2::Params params;
273 std::vector<ParameterizedOperator> defaultOperators;
278 void init(nlohmann::json config);
280 template <
typename Key>
283 if (cfg.
json().is_object() && cfg.
json().contains(key))
293 template <
typename Key>
296 return config<Key>(std::forward<Key>(key), m_config);
312 std::string fileSuffix()
const;
336 std::unique_ptr<detail::BufferedActions> >
339 std::map<std::string, adios2::Operator> m_operators;
344 filePositionToString(std::shared_ptr<ADIOS2FilePosition>)
override;
346 std::shared_ptr<ADIOS2FilePosition> extendFilePosition(
347 std::shared_ptr<ADIOS2FilePosition>
const &pos,
348 std::string extend)
override;
353 getCompressionOperator(std::string
const &compression);
359 std::string nameOfVariable(
Writable *writable);
370 std::string nameOfAttribute(
Writable *writable, std::string attribute);
376 ADIOS2FilePosition::GD groupOrDataset(
Writable *);
378 enum class IfFileNotOpen : bool
386 void dropFileData(InvalidatableFile file);
396 template <
typename T>
397 adios2::Variable<T> verifyDataset(
398 Offset
const &offset,
399 Extent
const &extent,
401 std::string
const &var);
408 namespace ADIOS2Defaults
410 using const_str =
char const *
const;
411 constexpr const_str str_engine =
"engine";
412 constexpr const_str str_type =
"type";
413 constexpr const_str str_params =
"parameters";
414 constexpr const_str str_usesteps =
"usesteps";
415 constexpr const_str str_usesstepsAttribute =
"__openPMD_internal/useSteps";
416 constexpr const_str str_adios2Schema =
417 "__openPMD_internal/openPMD2_adios2_schema";
418 constexpr const_str str_isBooleanOldLayout =
"__is_boolean__";
419 constexpr const_str str_isBooleanNewLayout =
420 "__openPMD_internal/is_boolean";
433 template <
typename T>
437 adios2::Engine &engine,
438 std::string
const &fileName);
440 std::string errorMsg =
"ADIOS2: readDataset()";
445 template <
typename T>
449 std::shared_ptr<Attribute::resource> resource);
451 template <
int n,
typename... Params>
457 template <
typename T>
463 template <
int n,
typename... Params>
464 void operator()(Params &&...);
469 template <
typename T>
474 std::shared_ptr<Attribute::resource> resource);
476 template <
int n,
typename... Params>
482 template <
typename T>
486 template <
int n,
typename... Params>
487 void operator()(Params &&...);
496 template <
typename T>
499 const std::string &varName,
502 std::string errorMsg =
"ADIOS2: openDataset()";
511 template <
typename T>
513 operator()(
BufferedPut &bp, adios2::IO &IO, adios2::Engine &engine);
515 template <
int n,
typename... Params>
516 void operator()(Params &&...);
536 template <
typename T>
539 std::string
const &name,
540 std::vector<ADIOS2IOHandlerImpl::ParameterizedOperator>
const 542 adios2::Dims
const &shape = adios2::Dims(),
543 adios2::Dims
const &start = adios2::Dims(),
544 adios2::Dims
const &count = adios2::Dims(),
545 bool const constantDims =
false);
547 std::string errorMsg =
"ADIOS2: defineVariable()";
552 template <
typename T>
556 adios2::Engine &engine,
557 std::string
const &varName);
559 template <
int n,
typename... Params>
560 void operator()(Params &&...);
571 template <
typename T>
575 oldCreateAttribute(adios2::IO &IO, std::string name,
T value);
577 static void oldReadAttribute(
580 std::shared_ptr<Attribute::resource> resource);
582 static void createAttribute(
584 adios2::Engine &engine,
588 static void readAttribute(
591 std::shared_ptr<Attribute::resource> resource);
599 auto attr = IO.InquireAttribute<
T>(name);
604 std::vector<T> data = attr.Data();
605 if (data.size() != 1)
609 return data[0] == val;
617 oldCreateAttribute(adios2::IO &, std::string, std::complex<long double>)
619 throw std::runtime_error(
620 "[ADIOS2] Internal error: no support for long double complex " 624 static void oldReadAttribute(
625 adios2::IO &, std::string, std::shared_ptr<Attribute::resource>)
627 throw std::runtime_error(
628 "[ADIOS2] Internal error: no support for long double complex " 632 static void createAttribute(
636 std::complex<long double>)
638 throw std::runtime_error(
639 "[ADIOS2] Internal error: no support for long double complex " 643 static void readAttribute(
646 std::shared_ptr<Attribute::resource>)
648 throw std::runtime_error(
649 "[ADIOS2] Internal error: no support for long double complex " 654 attributeUnchanged(adios2::IO &, std::string, std::complex<long double>)
656 throw std::runtime_error(
657 "[ADIOS2] Internal error: no support for long double complex " 665 static void oldCreateAttribute(
668 const std::vector<std::complex<long double> > &)
670 throw std::runtime_error(
671 "[ADIOS2] Internal error: no support for long double complex " 672 "vector attribute types");
675 static void oldReadAttribute(
676 adios2::IO &, std::string, std::shared_ptr<Attribute::resource>)
678 throw std::runtime_error(
679 "[ADIOS2] Internal error: no support for long double complex " 680 "vector attribute types");
683 static void createAttribute(
687 const std::vector<std::complex<long double> > &)
689 throw std::runtime_error(
690 "[ADIOS2] Internal error: no support for long double complex " 691 "vector attribute types");
694 static void readAttribute(
697 std::shared_ptr<Attribute::resource>)
699 throw std::runtime_error(
700 "[ADIOS2] Internal error: no support for long double complex " 701 "vector attribute types");
704 static bool attributeUnchanged(
705 adios2::IO &, std::string, std::vector<std::complex<long double> >)
707 throw std::runtime_error(
708 "[ADIOS2] Internal error: no support for long double complex " 709 "vector attribute types");
713 template <
typename T>
716 static void oldCreateAttribute(
717 adios2::IO &IO, std::string name,
const std::vector<T> &value);
719 static void oldReadAttribute(
722 std::shared_ptr<Attribute::resource> resource);
724 static void createAttribute(
726 adios2::Engine &engine,
728 const std::vector<T> &value);
730 static void readAttribute(
733 std::shared_ptr<Attribute::resource> resource);
736 attributeUnchanged(adios2::IO &IO, std::string name, std::vector<T> val)
738 auto attr = IO.InquireAttribute<
T>(name);
743 std::vector<T> data = attr.Data();
744 if (data.size() != val.size())
748 for (
size_t i = 0; i < val.size(); ++i)
750 if (data[i] != val[i])
762 static void oldCreateAttribute(
765 const std::vector<std::string> &value);
767 static void oldReadAttribute(
770 std::shared_ptr<Attribute::resource> resource);
772 static void createAttribute(
774 adios2::Engine &engine,
776 const std::vector<std::string> &vec);
778 static void readAttribute(
781 std::shared_ptr<Attribute::resource> resource);
783 static bool attributeUnchanged(
784 adios2::IO &IO, std::string name, std::vector<std::string> val)
786 auto attr = IO.InquireAttribute<std::string>(name);
791 std::vector<std::string> data = attr.Data();
792 if (data.size() != val.size())
796 for (
size_t i = 0; i < val.size(); ++i)
798 if (data[i] != val[i])
807 template <
typename T,
size_t n>
810 static void oldCreateAttribute(
811 adios2::IO &IO, std::string name,
const std::array<T, n> &value);
813 static void oldReadAttribute(
816 std::shared_ptr<Attribute::resource> resource);
818 static void createAttribute(
820 adios2::Engine &engine,
822 const std::array<T, n> &value);
824 static void readAttribute(
827 std::shared_ptr<Attribute::resource> resource);
829 static bool attributeUnchanged(
830 adios2::IO &IO, std::string name, std::array<T, n> val)
832 auto attr = IO.InquireAttribute<
T>(name);
837 std::vector<T> data = attr.Data();
838 if (data.size() != n)
842 for (
size_t i = 0; i < n; ++i)
844 if (data[i] != val[i])
856 using rep = detail::bool_representation;
859 oldCreateAttribute(adios2::IO &IO, std::string name,
bool value);
861 static void oldReadAttribute(
864 std::shared_ptr<Attribute::resource> resource);
866 static void createAttribute(
868 adios2::Engine &engine,
872 static void readAttribute(
875 std::shared_ptr<Attribute::resource> resource);
877 static constexpr rep toRep(
bool b)
882 static constexpr
bool fromRep(rep r)
888 attributeUnchanged(adios2::IO &IO, std::string name,
bool val)
890 auto attr = IO.InquireAttribute<rep>(name);
895 std::vector<rep> data = attr.Data();
896 if (data.size() != 1)
900 return data[0] == toRep(val);
961 Attribute::resource resource;
962 std::vector<char> bufferForVecString;
969 virtual void *update() = 0;
973 template <
typename T>
976 adios2::detail::Span<T> span;
980 void *update()
override;
1019 adios2::ADIOS &m_ADIOS;
1025 std::vector<std::unique_ptr<BufferedAction> >
m_buffer;
1050 adios2::Mode m_mode;
1070 std::set<std::string> uncommittedAttributes;
1082 bool optimizeAttributesStreaming =
false;
1084 using AttributeMap_t = std::map<std::string, adios2::Params>;
1096 adios2::Engine &getEngine();
1097 adios2::Engine &requireActiveStep();
1099 template <
typename BA>
1100 void enqueue(BA &&ba);
1102 template <
typename BA>
1103 void enqueue(BA &&ba, decltype(m_buffer) &);
1121 template <
typename F>
1124 F &&performPutsGets,
1125 bool writeAttributes,
1126 bool flushUnconditionally);
1133 void flush(
FlushLevel,
bool writeAttributes =
false);
1148 AttributeMap_t
const &availableAttributes();
1150 std::vector<std::string>
1151 availableAttributesPrefixed(std::string
const &prefix);
1156 void invalidateAttributesMap();
1158 AttributeMap_t
const &availableVariables();
1160 std::vector<std::string>
1161 availableVariablesPrefixed(std::string
const &prefix);
1166 void invalidateVariablesMap();
1174 std::string m_engineType;
1199 enum class StreamStatus
1264 StreamStatus streamStatus = StreamStatus::OutsideOfStep;
1265 adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK;
1272 bool delayOpeningTheFirstStep =
false;
1292 bool finalized =
false;
1294 inline SupportedSchema schema()
const 1296 return m_impl->schema();
1301 using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout;
1302 inline AttributeLayout attributeLayout()
const 1304 return m_impl->attributeLayout();
1309 #endif // openPMD_HAVE_ADIOS2 1313 #if openPMD_HAVE_ADIOS2 1326 this->flush(internal::defaultFlushParams);
1328 catch (std::exception
const &ex)
1330 std::cerr <<
"[~ADIOS2IOHandler] An error occurred: " << ex.what()
1335 std::cerr <<
"[~ADIOS2IOHandler] An error occurred." << std::endl;
1343 #if openPMD_HAVE_MPI 1349 nlohmann::json options,
1350 std::string engineType);
1357 nlohmann::json options,
1358 std::string engineType);
Definition: ADIOS2IOHandler.hpp:65
Wrapper around a shared pointer to:
Definition: InvalidatableFile.hpp:43
Definition: ADIOS2IOHandler.hpp:550
Definition: IOTask.hpp:562
FlushLevel
Determine what items should be flushed upon Series::flush()
Definition: AbstractIOHandler.hpp:66
Definition: ADIOS2IOHandler.hpp:967
Definition: ADIOS2IOHandler.hpp:949
Definition: ADIOS2IOHandler.hpp:455
Definition: ADIOS2IOHandler.hpp:467
Access
File access mode to use during IO.
Definition: Access.hpp:27
Definition: IOTask.hpp:144
Definition: ADIOS2IOHandler.hpp:519
Definition: IOTask.hpp:230
Definition: IOTask.hpp:333
std::string m_file
The full path to the file created on disk, including the containing directory and the file extension...
Definition: ADIOS2IOHandler.hpp:999
Definition: ADIOS2IOHandler.hpp:106
Definition: ADIOS2IOHandler.hpp:988
std::vector< std::unique_ptr< BufferedAction > > m_buffer
The default queue for deferred actions.
Definition: ADIOS2IOHandler.hpp:1025
Definition: ADIOS2IOHandler.hpp:933
Definition: ADIOS2IOHandler.cpp:646
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
Definition: ADIOS2IOHandler.hpp:77
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:33
Definition: ADIOS2IOHandler.hpp:941
std::string const m_IOName
ADIOS requires giving names to instances of adios2::IO.
Definition: ADIOS2IOHandler.hpp:1018
Definition: ADIOS2IOHandler.hpp:925
Definition: ADIOS2IOHandler.hpp:957
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:45
Definition: IOTask.hpp:120
Definition: Container.cpp:50
Definition: IOTask.hpp:159
Definition: IOTask.hpp:471
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:122
Simple Option type based on variantSrc::variant.
Definition: Option.hpp:45
std::map< unsigned, std::unique_ptr< I_UpdateSpan > > m_updateSpans
The base pointer of an ADIOS2 span might change after reallocations.
Definition: ADIOS2IOHandler.hpp:1059
Public definitions of openPMD-api.
Definition: Date.cpp:28
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:63
Definition: IOTask.hpp:193
Definition: ADIOS2IOHandler.hpp:1311
Definition: IOTask.hpp:384
Definition: ADIOS2IOHandler.hpp:505
std::string backendName() const override
The currently used backend.
Definition: ADIOS2IOHandler.hpp:1360
Definition: IOTask.hpp:543
Class that is responsible for scheduling and buffering openPMD attribute loads from ADIOS2...
Definition: ADIOS2PreloadAttributes.hpp:63
Definition: ADIOS2IOHandler.hpp:480
Parameters recursively passed through the openPMD hierarchy when flushing.
Definition: AbstractIOHandler.hpp:103
std::vector< BufferedAttributeRead > m_attributeReads
Definition: ADIOS2IOHandler.hpp:1042
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON.hpp:52
IterationEncoding
Encoding scheme of an Iterations Series'.
Definition: IterationEncoding.hpp:32
Definition: Writable.hpp:44
static bool attributeUnchanged(adios2::IO &IO, std::string name, T val)
Is the attribute given by parameters name and val already defined exactly in that way within the give...
Definition: ADIOS2IOHandler.hpp:597
Definition: ADIOS2IOHandler.hpp:490
std::map< std::string, BufferedAttributeWrite > m_attributeWrites
Buffer for attributes to be written in the new (variable-based) attribute layout. ...
Definition: ADIOS2IOHandler.hpp:1036
Definition: ADIOS2IOHandler.hpp:427
Definition: IOTask.hpp:101
Definition: IOTask.hpp:584
Definition: IOTask.hpp:418
Definition: IOTask.hpp:313
Definition: IOTask.hpp:176
Definition: ADIOS2IOHandler.hpp:974
Definition: IOTask.hpp:488
Definition: IOTask.hpp:265
nlohmann::json & json()
Access the underlying JSON value.
Definition: JSON.hpp:63
Definition: IOTask.hpp:436
Definition: ADIOS2IOHandler.hpp:73
Definition: ADIOS2IOHandler.hpp:911
Definition: IOTask.hpp:511
Definition: ADIOS2IOHandler.hpp:443
Definition: IOTask.hpp:296
Definition: IOTask.hpp:213
Definition: IOTask.hpp:350
std::vector< std::unique_ptr< BufferedAction > > m_alreadyEnqueued
This contains deferred actions that have already been enqueued into ADIOS2, but not yet performed in ...
Definition: ADIOS2IOHandler.hpp:1049
Definition: IOTask.hpp:247