23 #include "openPMD/IO/AbstractIOHandler.hpp" 24 #include "openPMD/IO/AbstractIOHandlerImpl.hpp" 25 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" 26 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" 27 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" 28 #include "openPMD/IO/IOTask.hpp" 29 #include "openPMD/IO/InvalidatableFile.hpp" 30 #include "openPMD/auxiliary/JSON.hpp" 31 #include "openPMD/auxiliary/Option.hpp" 32 #include "openPMD/backend/Writable.hpp" 33 #include "openPMD/config.hpp" 35 #if openPMD_HAVE_ADIOS2 41 #include <nlohmann/json.hpp> 50 #include <unordered_map> 57 #if openPMD_HAVE_ADIOS2 59 class ADIOS2IOHandler;
92 static constexpr
bool ADIOS2_DEBUG_MODE =
false;
102 nlohmann::json config,
103 std::string engineType );
105 #endif // openPMD_HAVE_MPI 109 nlohmann::json config,
110 std::string engineType );
115 std::future< void > flush( )
override;
190 adios2::Mode adios2AccessMode( );
194 adios2::ADIOS m_ADIOS;
198 std::string m_engineType;
200 struct ParameterizedOperator
203 adios2::Params params;
206 std::vector< ParameterizedOperator > defaultOperators;
212 init( nlohmann::json config );
214 template<
typename Key >
218 if( cfg.
json().is_object() && cfg.
json().contains( key ) )
228 template<
typename Key >
232 return config< Key >( std::forward< Key >( key ), m_config );
273 std::unique_ptr< detail::BufferedActions >
276 std::map< std::string, adios2::Operator > m_operators;
281 filePositionToString( std::shared_ptr< ADIOS2FilePosition > )
override;
283 std::shared_ptr< ADIOS2FilePosition >
284 extendFilePosition( std::shared_ptr< ADIOS2FilePosition >
const & pos,
285 std::string extend )
override;
290 getCompressionOperator( std::string
const & compression );
296 std::string nameOfVariable(
Writable * writable );
307 std::string nameOfAttribute(
Writable * writable, std::string attribute );
313 ADIOS2FilePosition::GD groupOrDataset(
Writable * );
317 void dropFileData( InvalidatableFile file );
327 template <
typename T >
328 adios2::Variable< T > verifyDataset( Offset
const & offset,
329 Extent
const & extent, adios2::IO & IO,
330 std::string
const & var );
337 namespace ADIOS2Defaults
339 using const_str =
char const *
const;
340 constexpr const_str str_engine =
"engine";
341 constexpr const_str str_type =
"type";
342 constexpr const_str str_params =
"parameters";
343 constexpr const_str str_usesteps =
"usesteps";
344 constexpr const_str str_usesstepsAttribute =
"__openPMD_internal/useSteps";
359 template <
typename T >
360 void operator( )(
BufferedGet & bp, adios2::IO & IO,
361 adios2::Engine & engine,
362 std::string
const & fileName );
364 template <
int T,
typename... Params >
void operator( )( Params &&... );
369 template <
typename T >
370 Datatype operator( )( adios2::IO & IO, std::string name,
371 std::shared_ptr< Attribute::resource > resource );
373 template <
int n,
typename... Params >
374 Datatype operator( )( Params &&... );
379 template <
typename T >
385 template <
int n,
typename... Params >
void operator( )( Params &&... );
396 template <
typename T >
401 template <
int n,
typename... Params >
void operator( )( Params &&... );
412 template <
typename T >
413 void operator( )(
BufferedPut & bp, adios2::IO & IO,
414 adios2::Engine & engine );
416 template <
int n,
typename... Params >
void operator( )( Params &&... );
422 template <
typename T,
typename... Params >
423 void operator( )( Params &&... params );
425 template<
int n,
typename... Params >
427 operator()( Params &&... );
432 template <
typename T,
typename... Params >
433 void operator( )( Params &&... );
435 template <
int n,
typename... Params >
436 void operator( )( Params &&... );
449 using Attr = adios2::Attribute< T >;
452 static Attr createAttribute( adios2::IO & IO, std::string name,
456 readAttribute( adios2::IO & IO, std::string name,
457 std::shared_ptr< Attribute::resource > resource );
466 auto attr = IO.InquireAttribute< BasicType >( name );
471 std::vector< BasicType > data = attr.Data();
472 if( data.size() != 1 )
476 return data[ 0 ] == val;
482 using Attr = adios2::Attribute< std::complex< double > >;
483 using BasicType = double;
485 static Attr createAttribute( adios2::IO &, std::string,
486 std::complex< long double > )
488 throw std::runtime_error(
489 "[ADIOS2] Internal error: no support for long double complex attribute types" );
493 readAttribute( adios2::IO &, std::string,
494 std::shared_ptr< Attribute::resource > )
496 throw std::runtime_error(
497 "[ADIOS2] Internal error: no support for long double complex attribute types" );
502 adios2::IO &, std::string, std::complex< long double > )
504 throw std::runtime_error(
505 "[ADIOS2] Internal error: no support for long double complex attribute types" );
511 using Attr = adios2::Attribute< std::complex< double > >;
512 using BasicType = double;
514 static Attr createAttribute( adios2::IO &, std::string,
515 const std::vector< std::complex< long double > > & )
517 throw std::runtime_error(
518 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
522 readAttribute( adios2::IO &, std::string,
523 std::shared_ptr< Attribute::resource > )
525 throw std::runtime_error(
526 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
533 std::vector< std::complex< long double > > )
535 throw std::runtime_error(
536 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
542 using Attr = adios2::Attribute< T >;
545 static Attr createAttribute( adios2::IO & IO, std::string name,
546 const std::vector< T > & value );
549 readAttribute( adios2::IO & IO, std::string name,
550 std::shared_ptr< Attribute::resource > resource );
556 std::vector< T > val )
558 auto attr = IO.InquireAttribute< BasicType >( name );
563 std::vector< BasicType > data = attr.Data();
564 if( data.size() != val.size() )
568 for(
size_t i = 0; i < val.size(); ++i )
570 if( data[ i ] != val[ i ] )
579 template <
typename T,
size_t n >
582 using Attr = adios2::Attribute< T >;
585 static Attr createAttribute( adios2::IO & IO, std::string name,
586 const std::array< T, n > & value );
589 readAttribute( adios2::IO & IO, std::string name,
590 std::shared_ptr< Attribute::resource > resource );
596 std::array< T, n > val )
598 auto attr = IO.InquireAttribute< BasicType >( name );
603 std::vector< BasicType > data = attr.Data();
604 if( data.size() != n )
608 for(
size_t i = 0; i < n; ++i )
610 if( data[ i ] != val[ i ] )
621 using rep = detail::bool_representation;
622 using Attr = adios2::Attribute< rep >;
623 using BasicType = rep;
625 static Attr createAttribute( adios2::IO & IO, std::string name,
629 readAttribute( adios2::IO & IO, std::string name,
630 std::shared_ptr< Attribute::resource > resource );
633 static constexpr rep toRep(
bool b )
639 static constexpr
bool fromRep( rep r )
645 attributeUnchanged( adios2::IO & IO, std::string name,
bool val )
647 auto attr = IO.InquireAttribute< BasicType >( name );
652 std::vector< BasicType > data = attr.Data();
653 if( data.size() != 1 )
657 return data[ 0 ] == toRep( val );
669 static constexpr
bool validType =
true;
674 static constexpr
bool validType =
false;
679 static constexpr
bool validType =
false;
685 static constexpr
bool validType =
false;
690 static constexpr
bool validType =
false;
701 template <
typename T >
703 T, typename
std::enable_if< DatasetTypes< T >::validType >::type >
714 void readDataset(
BufferedGet &, adios2::IO &, adios2::Engine &,
715 std::string
const & fileName );
735 std::string
const & name,
736 std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >
const &
738 adios2::Dims
const & shape = adios2::Dims(),
739 adios2::Dims
const & start = adios2::Dims(),
740 adios2::Dims
const & count = adios2::Dims(),
741 bool const constantDims =
false );
743 void writeDataset(
BufferedPut &, adios2::IO &, adios2::Engine & );
749 adios2::Engine & engine,
750 std::string
const & varName );
753 template <
typename T >
755 T, typename
std::enable_if< !DatasetTypes< T >::validType >::type >
760 static void throwErr( );
762 template <
typename... Params >
void openDataset( Params &&... );
764 template <
typename... Params >
void readDataset( Params &&... );
766 template <
typename... Params >
767 static void defineVariable( Params &&... );
769 template <
typename... Params >
void writeDataset( Params &&... );
771 template <
typename... Params >
static void blocksInfo( Params &&... );
849 adios2::ADIOS & m_ADIOS;
851 std::vector< std::unique_ptr< BufferedAction > > m_buffer;
862 std::set< std::string > uncommittedAttributes;
874 bool optimizeAttributesStreaming =
false;
876 using AttributeMap_t = std::map< std::string, adios2::Params >;
889 adios2::Engine & getEngine( );
890 adios2::Engine & requireActiveStep( );
892 template <
typename BA >
void enqueue( BA && ba );
894 template <
typename BA >
void enqueue( BA && ba, decltype( m_buffer ) & );
909 template<
typename F >
911 flush( F && performPutsGets,
bool flushUnconditionally );
935 AttributeMap_t
const &
936 availableAttributes();
938 std::vector< std::string >
939 availableAttributesPrefixed( std::string
const & prefix );
945 invalidateAttributesMap();
947 AttributeMap_t
const &
948 availableVariables();
950 std::vector< std::string >
951 availableVariablesPrefixed( std::string
const & prefix );
957 invalidateVariablesMap();
964 std::string m_engineType;
986 enum class StreamStatus
1034 StreamStatus streamStatus = StreamStatus::OutsideOfStep;
1035 adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK;
1042 bool delayOpeningTheFirstStep =
false;
1062 bool finalized =
false;
1070 #endif // openPMD_HAVE_ADIOS2 1075 #if openPMD_HAVE_ADIOS2 1090 catch( std::exception
const & ex )
1092 std::cerr <<
"[~ADIOS2IOHandler] An error occurred: " << ex.what() << std::endl;
1096 std::cerr <<
"[~ADIOS2IOHandler] An error occurred." << std::endl;
1104 #if openPMD_HAVE_MPI 1110 nlohmann::json options,
1111 std::string engineType );
1118 nlohmann::json options,
1119 std::string engineType );
1123 std::future< void > flush( )
override;
Definition: ADIOS2IOHandler.hpp:63
Wrapper around a shared pointer to:
Definition: InvalidatableFile.hpp:45
Definition: ADIOS2IOHandler.hpp:430
Definition: IOTask.hpp:493
Definition: ADIOS2IOHandler.hpp:805
Definition: ADIOS2IOHandler.hpp:367
Access
File access mode to use during IO.
Definition: Access.hpp:28
Definition: IOTask.hpp:139
static bool attributeUnchanged(adios2::IO &IO, std::string name, BasicType val)
Is the attribute given by parameters name and val already defined exactly in that way within the give...
Definition: ADIOS2IOHandler.hpp:464
Definition: ADIOS2IOHandler.hpp:419
Definition: IOTask.hpp:223
Definition: IOTask.hpp:320
std::string m_file
The full path to the file created on disk, including the containing directory and the file extension...
Definition: ADIOS2IOHandler.hpp:829
Definition: ADIOS2IOHandler.hpp:78
Definition: ADIOS2IOHandler.hpp:818
Definition: ADIOS2IOHandler.hpp:797
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
This struct's only field indicates whether the template parameter is a valid datatype to use for a da...
Definition: ADIOS2IOHandler.hpp:69
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:33
std::string const m_IOName
ADIOS requires giving names to instances of adios2::IO.
Definition: ADIOS2IOHandler.hpp:848
Definition: ADIOS2IOHandler.hpp:789
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:42
Definition: IOTask.hpp:123
Definition: Container.cpp:51
Definition: IOTask.hpp:153
Definition: IOTask.hpp:412
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:68
Simple Option type based on variantSrc::variant.
Definition: Option.hpp:47
Public definitions of openPMD-api.
Definition: Date.cpp:29
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:55
Definition: IOTask.hpp:185
Definition: ADIOS2IOHandler.hpp:1073
Definition: IOTask.hpp:365
Definition: ADIOS2IOHandler.hpp:404
std::string backendName() const override
The currently used backend.
Definition: ADIOS2IOHandler.hpp:1121
Definition: IOTask.hpp:475
Definition: ADIOS2IOHandler.hpp:377
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON.hpp:48
Definition: Writable.hpp:43
Definition: ADIOS2IOHandler.hpp:388
Definition: ADIOS2IOHandler.hpp:351
Definition: IOTask.hpp:107
Definition: IOTask.hpp:516
Definition: IOTask.hpp:394
Definition: IOTask.hpp:299
Definition: IOTask.hpp:169
Definition: IOTask.hpp:428
Definition: IOTask.hpp:256
nlohmann::json & json()
Access the underlying JSON value.
Definition: JSON.hpp:60
Definition: ADIOS2IOHandler.hpp:67
Definition: ADIOS2IOHandler.hpp:782
Definition: IOTask.hpp:447
Definition: IOTask.hpp:281
Definition: IOTask.hpp:207
Definition: IOTask.hpp:336
Definition: IOTask.hpp:239