openPMD-api
ADIOS2IOHandler.hpp
1 /* Copyright 2017-2020 Fabian Koller and Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/IO/AbstractIOHandler.hpp"
24 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
25 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
26 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
27 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
28 #include "openPMD/IO/IOTask.hpp"
29 #include "openPMD/IO/InvalidatableFile.hpp"
30 #include "openPMD/auxiliary/JSON.hpp"
31 #include "openPMD/auxiliary/Option.hpp"
32 #include "openPMD/backend/Writable.hpp"
33 #include "openPMD/config.hpp"
34 
35 #if openPMD_HAVE_ADIOS2
36 # include <adios2.h>
37 #endif
38 #if openPMD_HAVE_MPI
39 # include <mpi.h>
40 #endif
41 #include <nlohmann/json.hpp>
42 
43 #include <array>
44 #include <exception>
45 #include <future>
46 #include <iostream>
47 #include <memory> // shared_ptr
48 #include <set>
49 #include <string>
50 #include <unordered_map>
51 #include <utility> // pair
52 #include <vector>
53 
54 
55 namespace openPMD
56 {
57 #if openPMD_HAVE_ADIOS2
58 
59 class ADIOS2IOHandler;
60 
61 namespace detail
62 {
63  template < typename, typename > struct DatasetHelper;
64  struct DatasetReader;
65  struct AttributeReader;
66  struct AttributeWriter;
67  template < typename > struct AttributeTypes;
68  struct DatasetOpener;
69  template < typename > struct DatasetTypes;
70  struct WriteDataset;
71  struct BufferedActions;
72  struct BufferedPut;
73  struct BufferedGet;
74  struct BufferedAttributeRead;
75 } // namespace detail
76 
77 
79 : public AbstractIOHandlerImplCommon< ADIOS2FilePosition >
80 {
81  template < typename, typename > friend struct detail::DatasetHelper;
82  friend struct detail::DatasetReader;
83  friend struct detail::AttributeReader;
84  friend struct detail::AttributeWriter;
85  template < typename > friend struct detail::AttributeTypes;
86  friend struct detail::DatasetOpener;
87  template < typename > friend struct detail::DatasetTypes;
88  friend struct detail::WriteDataset;
89  friend struct detail::BufferedActions;
90  friend struct detail::BufferedAttributeRead;
91 
92  static constexpr bool ADIOS2_DEBUG_MODE = false;
93 
94 
95 public:
96 
97 #if openPMD_HAVE_MPI
98 
101  MPI_Comm,
102  nlohmann::json config,
103  std::string engineType );
104 
105 #endif // openPMD_HAVE_MPI
106 
107  explicit ADIOS2IOHandlerImpl(
109  nlohmann::json config,
110  std::string engineType );
111 
112 
113  ~ADIOS2IOHandlerImpl() override;
114 
115  std::future< void > flush( ) override;
116 
117  void createFile( Writable *,
118  Parameter< Operation::CREATE_FILE > const & ) override;
119 
120  void createPath( Writable *,
121  Parameter< Operation::CREATE_PATH > const & ) override;
122 
123  void
124  createDataset( Writable *,
125  Parameter< Operation::CREATE_DATASET > const & ) override;
126 
127  void
128  extendDataset( Writable *,
129  Parameter< Operation::EXTEND_DATASET > const & ) override;
130 
131  void openFile( Writable *,
132  Parameter< Operation::OPEN_FILE > const & ) override;
133 
134  void closeFile( Writable *,
135  Parameter< Operation::CLOSE_FILE > const & ) override;
136 
137  void openPath( Writable *,
138  Parameter< Operation::OPEN_PATH > const & ) override;
139 
140  void closePath( Writable *,
141  Parameter< Operation::CLOSE_PATH > const & ) override;
142 
143  void openDataset( Writable *,
145 
146  void deleteFile( Writable *,
147  Parameter< Operation::DELETE_FILE > const & ) override;
148 
149  void deletePath( Writable *,
150  Parameter< Operation::DELETE_PATH > const & ) override;
151 
152  void
153  deleteDataset( Writable *,
154  Parameter< Operation::DELETE_DATASET > const & ) override;
155 
156  void deleteAttribute( Writable *,
157  Parameter< Operation::DELETE_ATT > const & ) override;
158 
159  void writeDataset( Writable *,
160  Parameter< Operation::WRITE_DATASET > const & ) override;
161 
162  void writeAttribute( Writable *,
163  Parameter< Operation::WRITE_ATT > const & ) override;
164 
165  void readDataset( Writable *,
167 
168  void readAttribute( Writable *,
170 
171  void listPaths( Writable *, Parameter< Operation::LIST_PATHS > & ) override;
172 
173  void listDatasets( Writable *,
175 
176  void
177  listAttributes( Writable *,
178  Parameter< Operation::LIST_ATTS > & parameters ) override;
179 
180  void
181  advance( Writable*, Parameter< Operation::ADVANCE > & ) override;
182 
183  void
184  availableChunks( Writable*,
190  adios2::Mode adios2AccessMode( );
191 
192 
193 private:
194  adios2::ADIOS m_ADIOS;
198  std::string m_engineType;
199 
200  struct ParameterizedOperator
201  {
202  adios2::Operator op;
203  adios2::Params params;
204  };
205 
206  std::vector< ParameterizedOperator > defaultOperators;
207 
208  auxiliary::TracingJSON m_config;
209  static auxiliary::TracingJSON nullvalue;
210 
211  void
212  init( nlohmann::json config );
213 
214  template< typename Key >
216  config( Key && key, auxiliary::TracingJSON & cfg )
217  {
218  if( cfg.json().is_object() && cfg.json().contains( key ) )
219  {
220  return cfg[ key ];
221  }
222  else
223  {
224  return nullvalue;
225  }
226  }
227 
228  template< typename Key >
230  config( Key && key )
231  {
232  return config< Key >( std::forward< Key >( key ), m_config );
233  }
234 
243  getOperators( auxiliary::TracingJSON config );
244 
245  // use m_config
247  getOperators();
248 
249  std::string
250  fileSuffix() const;
251 
252  /*
253  * We need to give names to IO objects. These names are irrelevant
254  * within this application, since:
255  * 1) The name of the file written to is decided by the opened Engine's
256  * name.
257  * 2) The IOs are managed by the unordered_map m_fileData, so we do not
258  * need the ADIOS2 internal management.
259  * Since within one m_ADIOS object, the same IO name cannot be used more
260  * than once, we ensure different names by using the name counter.
261  * This allows to overwrite a file later without error.
262  */
263  int nameCounter{0};
264 
265  /*
266  * IO-heavy actions are deferred to a later point. This map stores for
267  * each open file (identified by an InvalidatableFile object) an object
268  * that manages IO-heavy actions, as well as its ADIOS2 objects, i.e.
269  * IO and Engine object.
270  * Not to be accessed directly, use getFileData().
271  */
272  std::unordered_map< InvalidatableFile,
273  std::unique_ptr< detail::BufferedActions >
274  > m_fileData;
275 
276  std::map< std::string, adios2::Operator > m_operators;
277 
278  // Overrides from AbstractIOHandlerImplCommon.
279 
280  std::string
281  filePositionToString( std::shared_ptr< ADIOS2FilePosition > ) override;
282 
283  std::shared_ptr< ADIOS2FilePosition >
284  extendFilePosition( std::shared_ptr< ADIOS2FilePosition > const & pos,
285  std::string extend ) override;
286 
287  // Helper methods.
288 
290  getCompressionOperator( std::string const & compression );
291 
292  /*
293  * The name of the ADIOS2 variable associated with this Writable.
294  * To be used for Writables that represent a dataset.
295  */
296  std::string nameOfVariable( Writable * writable );
297 
307  std::string nameOfAttribute( Writable * writable, std::string attribute );
308 
309  /*
310  * Figure out whether the Writable corresponds with a
311  * group or a dataset.
312  */
313  ADIOS2FilePosition::GD groupOrDataset( Writable * );
314 
315  detail::BufferedActions & getFileData( InvalidatableFile file );
316 
317  void dropFileData( InvalidatableFile file );
318 
319  /*
320  * Prepare a variable that already exists for an IO
321  * operation, including:
322  * (1) checking that its datatype matches T.
323  * (2) the offset and extent match the variable's shape
324  * (3) setting the offset and extent (ADIOS lingo: start
325  * and count)
326  */
327  template < typename T >
328  adios2::Variable< T > verifyDataset( Offset const & offset,
329  Extent const & extent, adios2::IO & IO,
330  std::string const & var );
331 }; // ADIOS2IOHandlerImpl
332 
333 /*
334  * The following strings are used during parsing of the JSON configuration
335  * string for the ADIOS2 backend.
336  */
337 namespace ADIOS2Defaults
338 {
339  using const_str = char const * const;
340  constexpr const_str str_engine = "engine";
341  constexpr const_str str_type = "type";
342  constexpr const_str str_params = "parameters";
343  constexpr const_str str_usesteps = "usesteps";
344  constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps";
345 } // namespace ADIOS2Defaults
346 
347 namespace detail
348 {
349  // Helper structs for calls to the switchType function
350 
352  {
354 
355 
356  explicit DatasetReader( openPMD::ADIOS2IOHandlerImpl * impl );
357 
358 
359  template < typename T >
360  void operator( )( BufferedGet & bp, adios2::IO & IO,
361  adios2::Engine & engine,
362  std::string const & fileName );
363 
364  template < int T, typename... Params > void operator( )( Params &&... );
365  };
366 
368  {
369  template < typename T >
370  Datatype operator( )( adios2::IO & IO, std::string name,
371  std::shared_ptr< Attribute::resource > resource );
372 
373  template < int n, typename... Params >
374  Datatype operator( )( Params &&... );
375  };
376 
378  {
379  template < typename T >
380  void
381  operator( )( ADIOS2IOHandlerImpl * impl, Writable * writable,
382  const Parameter< Operation::WRITE_ATT > & parameters );
383 
384 
385  template < int n, typename... Params > void operator( )( Params &&... );
386  };
387 
389  {
390  ADIOS2IOHandlerImpl * m_impl;
391 
392 
393  explicit DatasetOpener( ADIOS2IOHandlerImpl * impl );
394 
395 
396  template < typename T >
397  void operator( )( InvalidatableFile, const std::string & varName,
399 
400 
401  template < int n, typename... Params > void operator( )( Params &&... );
402  };
403 
405  {
406  ADIOS2IOHandlerImpl * m_handlerImpl;
407 
408 
409  WriteDataset( ADIOS2IOHandlerImpl * handlerImpl );
410 
411 
412  template < typename T >
413  void operator( )( BufferedPut & bp, adios2::IO & IO,
414  adios2::Engine & engine );
415 
416  template < int n, typename... Params > void operator( )( Params &&... );
417  };
418 
420  {
421  // Parameters such as DatasetHelper< T >::defineVariable
422  template < typename T, typename... Params >
423  void operator( )( Params &&... params );
424 
425  template< int n, typename... Params >
426  void
427  operator()( Params &&... );
428  };
429 
431  {
432  template < typename T, typename... Params >
433  void operator( )( Params &&... );
434 
435  template < int n, typename... Params >
436  void operator( )( Params &&... );
437  };
438 
439  // Helper structs to help distinguish valid attribute/variable
440  // datatypes from invalid ones
441 
442  /*
443  * This struct's purpose is to have specialisations
444  * for vector and array types, as well as the boolean
445  * type (which is not natively supported by ADIOS).
446  */
447  template < typename T > struct AttributeTypes
448  {
449  using Attr = adios2::Attribute< T >;
450  using BasicType = T;
451 
452  static Attr createAttribute( adios2::IO & IO, std::string name,
453  BasicType value );
454 
455  static void
456  readAttribute( adios2::IO & IO, std::string name,
457  std::shared_ptr< Attribute::resource > resource );
458 
463  static bool
464  attributeUnchanged( adios2::IO & IO, std::string name, BasicType val )
465  {
466  auto attr = IO.InquireAttribute< BasicType >( name );
467  if( !attr )
468  {
469  return false;
470  }
471  std::vector< BasicType > data = attr.Data();
472  if( data.size() != 1 )
473  {
474  return false;
475  }
476  return data[ 0 ] == val;
477  }
478  };
479 
480  template< > struct AttributeTypes< std::complex< long double > >
481  {
482  using Attr = adios2::Attribute< std::complex< double > >;
483  using BasicType = double;
484 
485  static Attr createAttribute( adios2::IO &, std::string,
486  std::complex< long double > )
487  {
488  throw std::runtime_error(
489  "[ADIOS2] Internal error: no support for long double complex attribute types" );
490  }
491 
492  static void
493  readAttribute( adios2::IO &, std::string,
494  std::shared_ptr< Attribute::resource > )
495  {
496  throw std::runtime_error(
497  "[ADIOS2] Internal error: no support for long double complex attribute types" );
498  }
499 
500  static bool
501  attributeUnchanged(
502  adios2::IO &, std::string, std::complex< long double > )
503  {
504  throw std::runtime_error(
505  "[ADIOS2] Internal error: no support for long double complex attribute types" );
506  }
507  };
508 
509  template< > struct AttributeTypes< std::vector< std::complex< long double > > >
510  {
511  using Attr = adios2::Attribute< std::complex< double > >;
512  using BasicType = double;
513 
514  static Attr createAttribute( adios2::IO &, std::string,
515  const std::vector< std::complex< long double > > & )
516  {
517  throw std::runtime_error(
518  "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
519  }
520 
521  static void
522  readAttribute( adios2::IO &, std::string,
523  std::shared_ptr< Attribute::resource > )
524  {
525  throw std::runtime_error(
526  "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
527  }
528 
529  static bool
530  attributeUnchanged(
531  adios2::IO &,
532  std::string,
533  std::vector< std::complex< long double > > )
534  {
535  throw std::runtime_error(
536  "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
537  }
538  };
539 
540  template < typename T > struct AttributeTypes< std::vector< T > >
541  {
542  using Attr = adios2::Attribute< T >;
543  using BasicType = T;
544 
545  static Attr createAttribute( adios2::IO & IO, std::string name,
546  const std::vector< T > & value );
547 
548  static void
549  readAttribute( adios2::IO & IO, std::string name,
550  std::shared_ptr< Attribute::resource > resource );
551 
552  static bool
553  attributeUnchanged(
554  adios2::IO & IO,
555  std::string name,
556  std::vector< T > val )
557  {
558  auto attr = IO.InquireAttribute< BasicType >( name );
559  if( !attr )
560  {
561  return false;
562  }
563  std::vector< BasicType > data = attr.Data();
564  if( data.size() != val.size() )
565  {
566  return false;
567  }
568  for( size_t i = 0; i < val.size(); ++i )
569  {
570  if( data[ i ] != val[ i ] )
571  {
572  return false;
573  }
574  }
575  return true;
576  }
577  };
578 
579  template < typename T, size_t n >
580  struct AttributeTypes< std::array< T, n > >
581  {
582  using Attr = adios2::Attribute< T >;
583  using BasicType = T;
584 
585  static Attr createAttribute( adios2::IO & IO, std::string name,
586  const std::array< T, n > & value );
587 
588  static void
589  readAttribute( adios2::IO & IO, std::string name,
590  std::shared_ptr< Attribute::resource > resource );
591 
592  static bool
593  attributeUnchanged(
594  adios2::IO & IO,
595  std::string name,
596  std::array< T, n > val )
597  {
598  auto attr = IO.InquireAttribute< BasicType >( name );
599  if( !attr )
600  {
601  return false;
602  }
603  std::vector< BasicType > data = attr.Data();
604  if( data.size() != n )
605  {
606  return false;
607  }
608  for( size_t i = 0; i < n; ++i )
609  {
610  if( data[ i ] != val[ i ] )
611  {
612  return false;
613  }
614  }
615  return true;
616  }
617  };
618 
619  template <> struct AttributeTypes< bool >
620  {
621  using rep = detail::bool_representation;
622  using Attr = adios2::Attribute< rep >;
623  using BasicType = rep;
624 
625  static Attr createAttribute( adios2::IO & IO, std::string name,
626  bool value );
627 
628  static void
629  readAttribute( adios2::IO & IO, std::string name,
630  std::shared_ptr< Attribute::resource > resource );
631 
632 
633  static constexpr rep toRep( bool b )
634  {
635  return b ? 1U : 0U;
636  }
637 
638 
639  static constexpr bool fromRep( rep r )
640  {
641  return r != 0;
642  }
643 
644  static bool
645  attributeUnchanged( adios2::IO & IO, std::string name, bool val )
646  {
647  auto attr = IO.InquireAttribute< BasicType >( name );
648  if( !attr )
649  {
650  return false;
651  }
652  std::vector< BasicType > data = attr.Data();
653  if( data.size() != 1 )
654  {
655  return false;
656  }
657  return data[ 0 ] == toRep( val );
658  }
659  };
660 
661 
667  template < typename T > struct DatasetTypes
668  {
669  static constexpr bool validType = true;
670  };
671 
672  template < typename T > struct DatasetTypes< std::vector< T > >
673  {
674  static constexpr bool validType = false;
675  };
676 
677  template <> struct DatasetTypes< bool >
678  {
679  static constexpr bool validType = false;
680  };
681 
682  // missing std::complex< long double > type in ADIOS2 v2.6.0
683  template <> struct DatasetTypes< std::complex< long double > >
684  {
685  static constexpr bool validType = false;
686  };
687 
688  template < typename T, size_t n > struct DatasetTypes< std::array< T, n > >
689  {
690  static constexpr bool validType = false;
691  };
692 
693  /*
694  * This struct's purpose is to have exactly two specialisations:
695  * (1) for types that are legal to use in a dataset
696  * (2) for types that are not legal to use in a dataset
697  * The methods in the latter specialisation will fail at runtime.
698  */
699  template < typename, typename = void > struct DatasetHelper;
700 
701  template < typename T >
703  T, typename std::enable_if< DatasetTypes< T >::validType >::type >
704  {
706 
707 
708  explicit DatasetHelper( openPMD::ADIOS2IOHandlerImpl * impl );
709 
710 
711  void openDataset( InvalidatableFile, const std::string & varName,
713 
714  void readDataset( BufferedGet &, adios2::IO &, adios2::Engine &,
715  std::string const & fileName );
716 
732  static void
733  defineVariable(
734  adios2::IO & IO,
735  std::string const & name,
736  std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const &
737  compressions,
738  adios2::Dims const & shape = adios2::Dims(),
739  adios2::Dims const & start = adios2::Dims(),
740  adios2::Dims const & count = adios2::Dims(),
741  bool const constantDims = false );
742 
743  void writeDataset( BufferedPut &, adios2::IO &, adios2::Engine & );
744 
745  static void
746  blocksInfo(
748  adios2::IO & IO,
749  adios2::Engine & engine,
750  std::string const & varName );
751  };
752 
753  template < typename T >
755  T, typename std::enable_if< !DatasetTypes< T >::validType >::type >
756  {
757  explicit DatasetHelper( openPMD::ADIOS2IOHandlerImpl * impl );
758 
759 
760  static void throwErr( );
761 
762  template < typename... Params > void openDataset( Params &&... );
763 
764  template < typename... Params > void readDataset( Params &&... );
765 
766  template < typename... Params >
767  static void defineVariable( Params &&... );
768 
769  template < typename... Params > void writeDataset( Params &&... );
770 
771  template < typename... Params > static void blocksInfo( Params &&... );
772  };
773 
774  // Other datatypes used in the ADIOS2IOHandler implementation
775 
776 
777  struct BufferedActions;
778 
779  /*
780  * IO-heavy action to be executed upon flushing.
781  */
783  {
784  virtual ~BufferedAction( ) = default;
785 
786  virtual void run( BufferedActions & ) = 0;
787  };
788 
790  {
791  std::string name;
793 
794  void run( BufferedActions & ) override;
795  };
796 
798  {
799  std::string name;
801 
802  void run( BufferedActions & ) override;
803  };
804 
806  {
808  std::string name;
809 
810  void run( BufferedActions & ) override;
811  };
812 
813  /*
814  * Manages per-file information about
815  * (1) the file's IO and Engine objects
816  * (2) the file's deferred IO-heavy actions
817  */
819  {
820  BufferedActions( BufferedActions const & ) = delete;
821 
829  std::string m_file;
848  std::string const m_IOName;
849  adios2::ADIOS & m_ADIOS;
850  adios2::IO m_IO;
851  std::vector< std::unique_ptr< BufferedAction > > m_buffer;
852  adios2::Mode m_mode;
853  detail::WriteDataset const m_writeDataset;
854  detail::DatasetReader const m_readDataset;
855  detail::AttributeReader const m_attributeReader;
856 
857  /*
858  * We call an attribute committed if the step during which it was
859  * written has been closed.
860  * A committed attribute cannot be modified.
861  */
862  std::set< std::string > uncommittedAttributes;
863 
864  /*
865  * The openPMD API will generally create new attributes for each
866  * iteration. This results in a growing number of attributes over time.
867  * In streaming-based modes, these will be completely sent anew in each
868  * iteration. If the following boolean is true, old attributes will be
869  * removed upon CLOSE_GROUP.
870  * Should not be set to true in persistent backends.
871  * Will be automatically set by BufferedActions::configure_IO depending
872  * on chosen ADIOS2 engine and can not be explicitly overridden by user.
873  */
874  bool optimizeAttributesStreaming = false;
875 
876  using AttributeMap_t = std::map< std::string, adios2::Params >;
877 
879 
880  ~BufferedActions( );
881 
886  void
887  finalize();
888 
889  adios2::Engine & getEngine( );
890  adios2::Engine & requireActiveStep( );
891 
892  template < typename BA > void enqueue( BA && ba );
893 
894  template < typename BA > void enqueue( BA && ba, decltype( m_buffer ) & );
895 
909  template< typename F >
910  void
911  flush( F && performPutsGets, bool flushUnconditionally );
912 
918  void
919  flush();
920 
928  advance( AdvanceMode mode );
929 
930  /*
931  * Delete all buffered actions without running them.
932  */
933  void drop( );
934 
935  AttributeMap_t const &
936  availableAttributes();
937 
938  std::vector< std::string >
939  availableAttributesPrefixed( std::string const & prefix );
940 
941  /*
942  * See description below.
943  */
944  void
945  invalidateAttributesMap();
946 
947  AttributeMap_t const &
948  availableVariables();
949 
950  std::vector< std::string >
951  availableVariablesPrefixed( std::string const & prefix );
952 
953  /*
954  * See description below.
955  */
956  void
957  invalidateVariablesMap();
958 
959  private:
961 
964  std::string m_engineType;
965  /*
966  * streamStatus is NoStream for file-based ADIOS engines.
967  * This is relevant for the method BufferedActions::requireActiveStep,
968  * where a step is only opened if the status is OutsideOfStep, but not
969  * if NoStream. The rationale behind this is that parsing a Series
970  * works differently for file-based and for stream-based engines:
971  * * stream-based: Iterations are parsed as they arrive. For parsing an
972  * iteration, the iteration must be awaited.
973  * BufferedActions::requireActiveStep takes care of this.
974  * * file-based: The Series is parsed up front. If no step has been
975  * opened yet, ADIOS2 gives access to all variables and attributes
976  * from all steps. Upon opening a step, only the variables from that
977  * step are shown which hinders parsing. So, until a step is
978  * explicitly opened via ADIOS2IOHandlerImpl::advance, do not open
979  * one.
980  * (This is a workaround for the fact that attributes
981  * are not associated with steps in ADIOS -- seeing all attributes
982  * from all steps in file-based engines, but only the current
983  * variables breaks parsing otherwise.)
984  *
985  */
986  enum class StreamStatus
987  {
991  DuringStep,
995  OutsideOfStep,
999  StreamOver,
1011  NoStream,
1026  Parsing,
1032  Undecided
1033  };
1034  StreamStatus streamStatus = StreamStatus::OutsideOfStep;
1035  adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK;
1036 
1042  bool delayOpeningTheFirstStep = false;
1043 
1044  /*
1045  * ADIOS2 does not give direct access to its internal attribute and
1046  * variable maps, but will instead give access to copies of them.
1047  * In order to avoid unnecessary copies, we buffer the returned map.
1048  * The downside of this is that we need to pay attention to invalidate
1049  * the map whenever an attribute/variable is altered. In that case, we
1050  * fetch the map anew.
1051  * If empty, the buffered map has been invalidated and needs to be
1052  * queried from ADIOS2 again. If full, the buffered map is equivalent to
1053  * the map that would be returned by a call to
1054  * IO::Available(Attributes|Variables).
1055  */
1056  auxiliary::Option< AttributeMap_t > m_availableAttributes;
1057  auxiliary::Option< AttributeMap_t > m_availableVariables;
1058 
1059  /*
1060  * finalize() will set this true to avoid running twice.
1061  */
1062  bool finalized = false;
1063 
1064  void
1065  configure_IO( ADIOS2IOHandlerImpl & impl );
1066  };
1067 
1068 
1069 } // namespace detail
1070 #endif // openPMD_HAVE_ADIOS2
1071 
1072 
1074 {
1075 #if openPMD_HAVE_ADIOS2
1076 
1077 friend class ADIOS2IOHandlerImpl;
1078 
1079 private:
1080  ADIOS2IOHandlerImpl m_impl;
1081 
1082 public:
1083  ~ADIOS2IOHandler( ) override
1084  {
1085  // we must not throw in a destructor
1086  try
1087  {
1088  this->flush( );
1089  }
1090  catch( std::exception const & ex )
1091  {
1092  std::cerr << "[~ADIOS2IOHandler] An error occurred: " << ex.what() << std::endl;
1093  }
1094  catch( ... )
1095  {
1096  std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
1097  }
1098  }
1099 
1100 #else
1101 public:
1102 #endif
1103 
1104 #if openPMD_HAVE_MPI
1105 
1107  std::string path,
1108  Access,
1109  MPI_Comm,
1110  nlohmann::json options,
1111  std::string engineType );
1112 
1113 #endif
1114 
1116  std::string path,
1117  Access,
1118  nlohmann::json options,
1119  std::string engineType );
1120 
1121  std::string backendName() const override { return "ADIOS2"; }
1122 
1123  std::future< void > flush( ) override;
1124 }; // ADIOS2IOHandler
1125 } // namespace openPMD
Definition: ADIOS2IOHandler.hpp:63
Wrapper around a shared pointer to:
Definition: InvalidatableFile.hpp:45
Definition: ADIOS2IOHandler.hpp:430
Definition: ADIOS2IOHandler.hpp:805
Definition: ADIOS2IOHandler.hpp:367
Access
File access mode to use during IO.
Definition: Access.hpp:28
static bool attributeUnchanged(adios2::IO &IO, std::string name, BasicType val)
Is the attribute given by parameters name and val already defined exactly in that way within the give...
Definition: ADIOS2IOHandler.hpp:464
Definition: ADIOS2IOHandler.hpp:419
STL namespace.
std::string m_file
The full path to the file created on disk, including the containing directory and the file extension...
Definition: ADIOS2IOHandler.hpp:829
Definition: ADIOS2IOHandler.hpp:78
Definition: ADIOS2IOHandler.hpp:818
Definition: ADIOS2IOHandler.hpp:797
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
This struct&#39;s only field indicates whether the template parameter is a valid datatype to use for a da...
Definition: ADIOS2IOHandler.hpp:69
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:33
std::string const m_IOName
ADIOS requires giving names to instances of adios2::IO.
Definition: ADIOS2IOHandler.hpp:848
Definition: ADIOS2IOHandler.hpp:789
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:42
Definition: Container.cpp:51
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:68
Simple Option type based on variantSrc::variant.
Definition: Option.hpp:47
Public definitions of openPMD-api.
Definition: Date.cpp:29
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:55
Definition: ADIOS2IOHandler.hpp:1073
Definition: ADIOS2IOHandler.hpp:404
std::string backendName() const override
The currently used backend.
Definition: ADIOS2IOHandler.hpp:1121
Definition: ADIOS2IOHandler.hpp:377
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON.hpp:48
Definition: Writable.hpp:43
Definition: ADIOS2IOHandler.hpp:388
Definition: ADIOS2IOHandler.hpp:351
nlohmann::json & json()
Access the underlying JSON value.
Definition: JSON.hpp:60
Definition: ADIOS2IOHandler.hpp:67
Definition: ADIOS2IOHandler.hpp:782