23 #include "openPMD/IO/AbstractIOHandler.hpp" 24 #include "openPMD/IO/AbstractIOHandlerImpl.hpp" 25 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" 26 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" 27 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" 28 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" 29 #include "openPMD/IO/IOTask.hpp" 30 #include "openPMD/IO/InvalidatableFile.hpp" 31 #include "openPMD/auxiliary/JSON.hpp" 32 #include "openPMD/auxiliary/Option.hpp" 33 #include "openPMD/backend/Writable.hpp" 34 #include "openPMD/config.hpp" 35 #include "openPMD/IterationEncoding.hpp" 37 #if openPMD_HAVE_ADIOS2 43 #include <nlohmann/json.hpp> 52 #include <unordered_map> 59 #if openPMD_HAVE_ADIOS2 61 class ADIOS2IOHandler;
85 namespace ADIOS2Schema
87 using schema_t = uint64_t;
91 constexpr schema_t schema_0000_00_00 = 00000000;
95 constexpr schema_t schema_2021_02_09 = 20210209;
97 enum class SupportedSchema : char
103 using SupportedSchema = ADIOS2Schema::SupportedSchema;
123 static constexpr
bool ADIOS2_DEBUG_MODE =
false;
133 nlohmann::json config,
134 std::string engineType );
136 #endif // openPMD_HAVE_MPI 140 nlohmann::json config,
141 std::string engineType );
146 std::future< void > flush( )
override;
224 adios2::Mode adios2AccessMode( std::string
const & fullPath );
227 adios2::ADIOS m_ADIOS;
236 std::string m_engineType;
237 ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00;
239 enum class AttributeLayout : char
245 inline SupportedSchema schema()
const 249 case ADIOS2Schema::schema_0000_00_00:
250 return SupportedSchema::s_0000_00_00;
251 case ADIOS2Schema::schema_2021_02_09:
252 return SupportedSchema::s_2021_02_09;
254 throw std::runtime_error(
255 "[ADIOS2] Encountered unsupported schema version: " +
256 std::to_string( m_schema ) );
260 inline AttributeLayout attributeLayout()
const 264 case SupportedSchema::s_0000_00_00:
265 return AttributeLayout::ByAdiosAttributes;
266 case SupportedSchema::s_2021_02_09:
267 return AttributeLayout::ByAdiosVariables;
269 throw std::runtime_error(
"Unreachable!" );
272 struct ParameterizedOperator
275 adios2::Params params;
278 std::vector< ParameterizedOperator > defaultOperators;
284 init( nlohmann::json config );
286 template<
typename Key >
290 if( cfg.
json().is_object() && cfg.
json().contains( key ) )
300 template<
typename Key >
304 return config< Key >( std::forward< Key >( key ), m_config );
345 std::unique_ptr< detail::BufferedActions >
348 std::map< std::string, adios2::Operator > m_operators;
353 filePositionToString( std::shared_ptr< ADIOS2FilePosition > )
override;
355 std::shared_ptr< ADIOS2FilePosition >
356 extendFilePosition( std::shared_ptr< ADIOS2FilePosition >
const & pos,
357 std::string extend )
override;
362 getCompressionOperator( std::string
const & compression );
368 std::string nameOfVariable(
Writable * writable );
379 std::string nameOfAttribute(
Writable * writable, std::string attribute );
385 ADIOS2FilePosition::GD groupOrDataset(
Writable * );
387 enum class IfFileNotOpen : bool
394 getFileData( InvalidatableFile file, IfFileNotOpen );
396 void dropFileData( InvalidatableFile file );
406 template <
typename T >
407 adios2::Variable< T > verifyDataset( Offset
const & offset,
408 Extent
const & extent, adios2::IO & IO,
409 std::string
const & var );
416 namespace ADIOS2Defaults
418 using const_str =
char const *
const;
419 constexpr const_str str_engine =
"engine";
420 constexpr const_str str_type =
"type";
421 constexpr const_str str_params =
"parameters";
422 constexpr const_str str_usesteps =
"usesteps";
423 constexpr const_str str_usesstepsAttribute =
"__openPMD_internal/useSteps";
424 constexpr const_str str_adios2Schema =
425 "__openPMD_internal/openPMD2_adios2_schema";
426 constexpr const_str str_isBooleanOldLayout =
"__is_boolean__";
427 constexpr const_str str_isBooleanNewLayout =
428 "__openPMD_internal/is_boolean";
443 template <
typename T >
444 void operator( )(
BufferedGet & bp, adios2::IO & IO,
445 adios2::Engine & engine,
446 std::string
const & fileName );
448 std::string errorMsg =
"ADIOS2: readDataset()";
453 template<
typename T >
458 std::shared_ptr< Attribute::resource > resource );
460 template<
int n,
typename... Params >
462 operator()( Params &&... );
467 template<
typename T >
475 template<
int n,
typename... Params >
477 operator()( Params &&... );
482 template<
typename T >
488 std::shared_ptr< Attribute::resource > resource );
490 template <
int n,
typename... Params >
491 Datatype operator( )( Params &&... );
496 template <
typename T >
503 template <
int n,
typename... Params >
void operator( )( Params &&... );
513 template <
typename T >
518 std::string errorMsg =
"ADIOS2: openDataset()";
529 template <
typename T >
530 void operator( )(
BufferedPut & bp, adios2::IO & IO,
531 adios2::Engine & engine );
533 template <
int n,
typename... Params >
void operator( )( Params &&... );
553 template <
typename T >
556 std::string
const & name,
557 std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >
const &
559 adios2::Dims
const & shape = adios2::Dims(),
560 adios2::Dims
const & start = adios2::Dims(),
561 adios2::Dims
const & count = adios2::Dims(),
562 bool const constantDims = false );
564 std::string errorMsg =
"ADIOS2: defineVariable()";
569 template <
typename T >
573 adios2::Engine & engine,
574 std::string
const & varName );
576 template <
int n,
typename... Params >
577 void operator( )( Params &&... );
588 template<
typename T >
601 std::shared_ptr< Attribute::resource > resource );
606 adios2::Engine & engine,
614 std::shared_ptr< Attribute::resource > resource );
623 auto attr = IO.InquireAttribute<
T >( name );
628 std::vector< T > data = attr.Data();
629 if( data.size() != 1 )
633 return data[ 0 ] == val;
643 std::complex< long double > )
645 throw std::runtime_error(
646 "[ADIOS2] Internal error: no support for long double complex attribute types" );
653 std::shared_ptr< Attribute::resource > )
655 throw std::runtime_error(
656 "[ADIOS2] Internal error: no support for long double complex attribute types" );
664 std::complex< long double > )
666 throw std::runtime_error(
667 "[ADIOS2] Internal error: no support for long double complex attribute types" );
674 std::shared_ptr< Attribute::resource > )
676 throw std::runtime_error(
677 "[ADIOS2] Internal error: no support for long double complex attribute types" );
682 adios2::IO &, std::string, std::complex< long double > )
684 throw std::runtime_error(
685 "[ADIOS2] Internal error: no support for long double complex attribute types" );
695 const std::vector< std::complex< long double > > & )
697 throw std::runtime_error(
698 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
705 std::shared_ptr< Attribute::resource > )
707 throw std::runtime_error(
708 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
716 const std::vector< std::complex< long double > > & )
718 throw std::runtime_error(
719 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
726 std::shared_ptr< Attribute::resource > )
728 throw std::runtime_error(
729 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
736 std::vector< std::complex< long double > > )
738 throw std::runtime_error(
739 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
749 const std::vector< T > & value );
755 std::shared_ptr< Attribute::resource > resource );
760 adios2::Engine & engine,
762 const std::vector< T > & value );
768 std::shared_ptr< Attribute::resource > resource );
774 std::vector< T > val )
776 auto attr = IO.InquireAttribute<
T >( name );
781 std::vector< T > data = attr.Data();
782 if( data.size() != val.size() )
786 for(
size_t i = 0; i < val.size(); ++i )
788 if( data[ i ] != val[ i ] )
804 const std::vector< std::string > & value );
810 std::shared_ptr< Attribute::resource > resource );
815 adios2::Engine & engine,
817 const std::vector< std::string > & vec );
823 std::shared_ptr< Attribute::resource > resource );
829 std::vector< std::string > val )
831 auto attr = IO.InquireAttribute< std::string >( name );
836 std::vector< std::string > data = attr.Data();
837 if( data.size() != val.size() )
841 for(
size_t i = 0; i < val.size(); ++i )
843 if( data[ i ] != val[ i ] )
852 template <
typename T,
size_t n >
859 const std::array< T, n > & value );
865 std::shared_ptr< Attribute::resource > resource );
870 adios2::Engine & engine,
872 const std::array< T, n > & value );
878 std::shared_ptr< Attribute::resource > resource );
884 std::array< T, n > val )
886 auto attr = IO.InquireAttribute<
T >( name );
891 std::vector< T > data = attr.Data();
892 if( data.size() != n )
896 for(
size_t i = 0; i < n; ++i )
898 if( data[ i ] != val[ i ] )
909 using rep = detail::bool_representation;
912 oldCreateAttribute( adios2::IO & IO, std::string name,
bool value );
918 std::shared_ptr< Attribute::resource > resource );
923 adios2::Engine & engine,
931 std::shared_ptr< Attribute::resource > resource );
934 static constexpr rep toRep(
bool b )
940 static constexpr
bool fromRep( rep r )
946 attributeUnchanged( adios2::IO & IO, std::string name,
bool val )
948 auto attr = IO.InquireAttribute< rep >( name );
953 std::vector< rep > data = attr.Data();
954 if( data.size() != 1 )
958 return data[ 0 ] == toRep( val );
1021 Attribute::resource resource;
1022 std::vector< char > bufferForVecString;
1029 virtual void *update() = 0;
1033 template<
typename T >
1036 adios2::detail::Span< T > span;
1040 void *update()
override;
1079 adios2::ADIOS & m_ADIOS;
1085 std::vector< std::unique_ptr< BufferedAction > >
m_buffer;
1110 adios2::Mode m_mode;
1130 std::set< std::string > uncommittedAttributes;
1142 bool optimizeAttributesStreaming =
false;
1144 using AttributeMap_t = std::map< std::string, adios2::Params >;
1157 adios2::Engine & getEngine( );
1158 adios2::Engine & requireActiveStep( );
1160 template <
typename BA >
void enqueue( BA && ba );
1162 template <
typename BA >
void enqueue( BA && ba, decltype( m_buffer ) & );
1180 template<
typename F >
1184 F && performPutsGets,
1185 bool writeAttributes,
1186 bool flushUnconditionally );
1194 flush(
FlushLevel,
bool writeAttributes =
false );
1210 AttributeMap_t
const &
1211 availableAttributes();
1213 std::vector< std::string >
1214 availableAttributesPrefixed( std::string
const & prefix );
1220 invalidateAttributesMap();
1222 AttributeMap_t
const &
1223 availableVariables();
1225 std::vector< std::string >
1226 availableVariablesPrefixed( std::string
const & prefix );
1232 invalidateVariablesMap();
1240 std::string m_engineType;
1265 enum class StreamStatus
1330 StreamStatus streamStatus = StreamStatus::OutsideOfStep;
1331 adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK;
1338 bool delayOpeningTheFirstStep =
false;
1358 bool finalized =
false;
1360 inline SupportedSchema schema()
const 1362 return m_impl->schema();
1368 using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout;
1369 inline AttributeLayout attributeLayout()
const 1371 return m_impl->attributeLayout();
1376 #endif // openPMD_HAVE_ADIOS2 1381 #if openPMD_HAVE_ADIOS2 1396 catch( std::exception
const & ex )
1398 std::cerr <<
"[~ADIOS2IOHandler] An error occurred: " << ex.what() << std::endl;
1402 std::cerr <<
"[~ADIOS2IOHandler] An error occurred." << std::endl;
1410 #if openPMD_HAVE_MPI 1416 nlohmann::json options,
1417 std::string engineType );
1424 nlohmann::json options,
1425 std::string engineType );
1429 std::future< void > flush( )
override;
Definition: ADIOS2IOHandler.hpp:65
Wrapper around a shared pointer to:
Definition: InvalidatableFile.hpp:45
Definition: ADIOS2IOHandler.hpp:567
Definition: IOTask.hpp:532
Definition: ADIOS2IOHandler.hpp:1027
std::map< unsigned, std::unique_ptr< I_UpdateSpan > > m_updateSpans
The base pointer of an ADIOS2 span might change after reallocations.
Definition: ADIOS2IOHandler.hpp:1119
Definition: ADIOS2IOHandler.hpp:1008
Definition: ADIOS2IOHandler.hpp:465
Definition: ADIOS2IOHandler.hpp:480
Access
File access mode to use during IO.
Definition: Access.hpp:28
Definition: IOTask.hpp:149
Definition: ADIOS2IOHandler.hpp:536
Definition: IOTask.hpp:233
Definition: IOTask.hpp:328
std::string m_file
The full path to the file created on disk, including the containing directory and the file extension...
Definition: ADIOS2IOHandler.hpp:1059
Definition: ADIOS2IOHandler.hpp:105
Definition: ADIOS2IOHandler.hpp:1048
FlushLevel
Determine what items should be flushed upon Series::flush()
Definition: AbstractIOHandler.hpp:63
Definition: ADIOS2IOHandler.hpp:992
Definition: ADIOS2IOHandler.cpp:661
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
Definition: ADIOS2IOHandler.hpp:75
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:33
std::vector< std::unique_ptr< BufferedAction > > m_buffer
The default queue for deferred actions.
Definition: ADIOS2IOHandler.hpp:1085
Definition: ADIOS2IOHandler.hpp:1000
std::string const m_IOName
ADIOS requires giving names to instances of adios2::IO.
Definition: ADIOS2IOHandler.hpp:1078
Definition: ADIOS2IOHandler.hpp:984
Definition: ADIOS2IOHandler.hpp:1017
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:45
Definition: IOTask.hpp:126
Definition: Container.cpp:51
Definition: IOTask.hpp:163
Definition: IOTask.hpp:451
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:98
Simple Option type based on variantSrc::variant.
Definition: Option.hpp:47
Public definitions of openPMD-api.
Definition: Date.cpp:29
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:64
Definition: IOTask.hpp:195
Definition: ADIOS2IOHandler.hpp:1379
Definition: IOTask.hpp:373
Definition: ADIOS2IOHandler.hpp:521
std::string backendName() const override
The currently used backend.
Definition: ADIOS2IOHandler.hpp:1427
Definition: IOTask.hpp:514
Class that is responsible for scheduling and buffering openPMD attribute loads from ADIOS2...
Definition: ADIOS2PreloadAttributes.hpp:63
std::vector< BufferedAttributeRead > m_attributeReads
Definition: ADIOS2IOHandler.hpp:1102
Definition: ADIOS2IOHandler.hpp:494
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON.hpp:52
IterationEncoding
Encoding scheme of an Iterations Series'.
Definition: IterationEncoding.hpp:32
Definition: Writable.hpp:45
static bool attributeUnchanged(adios2::IO &IO, std::string name, T val)
Is the attribute given by parameters name and val already defined exactly in that way within the give...
Definition: ADIOS2IOHandler.hpp:621
Definition: ADIOS2IOHandler.hpp:506
Definition: ADIOS2IOHandler.hpp:435
Definition: IOTask.hpp:108
Definition: IOTask.hpp:555
Definition: IOTask.hpp:402
std::vector< std::unique_ptr< BufferedAction > > m_alreadyEnqueued
This contains deferred actions that have already been enqueued into ADIOS2, but not yet performed in ...
Definition: ADIOS2IOHandler.hpp:1109
std::map< std::string, BufferedAttributeWrite > m_attributeWrites
Buffer for attributes to be written in the new (variable-based) attribute layout. ...
Definition: ADIOS2IOHandler.hpp:1096
Definition: IOTask.hpp:307
Definition: IOTask.hpp:179
Definition: ADIOS2IOHandler.hpp:1034
Definition: IOTask.hpp:467
Definition: IOTask.hpp:266
nlohmann::json & json()
Access the underlying JSON value.
Definition: JSON.hpp:64
Definition: IOTask.hpp:420
Definition: ADIOS2IOHandler.hpp:72
Definition: ADIOS2IOHandler.hpp:970
Definition: IOTask.hpp:486
Definition: ADIOS2IOHandler.hpp:451
Definition: IOTask.hpp:291
Definition: IOTask.hpp:217
Definition: IOTask.hpp:344
Definition: IOTask.hpp:249