openPMD-api
ADIOS2IOHandler.hpp
1 /* Copyright 2017-2020 Fabian Koller and Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "ADIOS2FilePosition.hpp"
24 #include "openPMD/config.hpp"
25 #include "openPMD/IO/AbstractIOHandler.hpp"
26 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
27 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
28 #include "openPMD/IO/IOTask.hpp"
29 #include "openPMD/IO/InvalidatableFile.hpp"
30 #include "openPMD/auxiliary/JSON.hpp"
31 #include "openPMD/backend/Writable.hpp"
32 
33 #if openPMD_HAVE_ADIOS2
34 # include <adios2.h>
35 # include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
36 #endif
37 
38 #if openPMD_HAVE_MPI
39 # include <mpi.h>
40 #endif
41 
42 #include <nlohmann/json.hpp>
43 
44 #include <array>
45 #include <exception>
46 #include <future>
47 #include <iostream>
48 #include <memory> // shared_ptr
49 #include <string>
50 #include <unordered_map>
51 #include <utility> // pair
52 #include <vector>
53 
54 namespace openPMD
55 {
56 #if openPMD_HAVE_ADIOS2
57 
58 class ADIOS2IOHandler;
59 
60 namespace detail
61 {
62  template < typename, typename > struct DatasetHelper;
63  struct DatasetReader;
64  struct AttributeReader;
65  struct AttributeWriter;
66  template < typename > struct AttributeTypes;
67  struct DatasetOpener;
68  template < typename > struct DatasetTypes;
69  struct WriteDataset;
70  struct BufferedActions;
71  struct BufferedPut;
72  struct BufferedGet;
73  struct BufferedAttributeRead;
74 } // namespace detail
75 
76 
78 : public AbstractIOHandlerImplCommon< ADIOS2FilePosition >
79 {
80  template < typename, typename > friend struct detail::DatasetHelper;
81  friend struct detail::DatasetReader;
82  friend struct detail::AttributeReader;
83  friend struct detail::AttributeWriter;
84  template < typename > friend struct detail::AttributeTypes;
85  friend struct detail::DatasetOpener;
86  template < typename > friend struct detail::DatasetTypes;
87  friend struct detail::WriteDataset;
88  friend struct detail::BufferedActions;
89  friend struct detail::BufferedAttributeRead;
90 
91  static constexpr bool ADIOS2_DEBUG_MODE = false;
92 
93 
94 public:
95 
96 #if openPMD_HAVE_MPI
97 
98  ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, nlohmann::json config );
99 
100  MPI_Comm m_comm;
101 
102 #endif // openPMD_HAVE_MPI
103 
104  explicit ADIOS2IOHandlerImpl(
106  , nlohmann::json config
107  );
108 
109 
110  ~ADIOS2IOHandlerImpl( ) override = default;
111 
112  std::future< void > flush( ) override;
113 
114  void createFile( Writable *,
115  Parameter< Operation::CREATE_FILE > const & ) override;
116 
117  void createPath( Writable *,
118  Parameter< Operation::CREATE_PATH > const & ) override;
119 
120  void
121  createDataset( Writable *,
122  Parameter< Operation::CREATE_DATASET > const & ) override;
123 
124  void
125  extendDataset( Writable *,
126  Parameter< Operation::EXTEND_DATASET > const & ) override;
127 
128  void openFile( Writable *,
129  Parameter< Operation::OPEN_FILE > const & ) override;
130 
131  void closeFile( Writable *,
132  Parameter< Operation::CLOSE_FILE > const & ) override;
133 
134  void openPath( Writable *,
135  Parameter< Operation::OPEN_PATH > const & ) override;
136 
137  void openDataset( Writable *,
139 
140  void deleteFile( Writable *,
141  Parameter< Operation::DELETE_FILE > const & ) override;
142 
143  void deletePath( Writable *,
144  Parameter< Operation::DELETE_PATH > const & ) override;
145 
146  void
147  deleteDataset( Writable *,
148  Parameter< Operation::DELETE_DATASET > const & ) override;
149 
150  void deleteAttribute( Writable *,
151  Parameter< Operation::DELETE_ATT > const & ) override;
152 
153  void writeDataset( Writable *,
154  Parameter< Operation::WRITE_DATASET > const & ) override;
155 
156  void writeAttribute( Writable *,
157  Parameter< Operation::WRITE_ATT > const & ) override;
158 
159  void readDataset( Writable *,
161 
162  void readAttribute( Writable *,
164 
165  void listPaths( Writable *, Parameter< Operation::LIST_PATHS > & ) override;
166 
167  void listDatasets( Writable *,
169 
170  void
171  listAttributes( Writable *,
172  Parameter< Operation::LIST_ATTS > & parameters ) override;
173 
178  adios2::Mode adios2AccessMode( );
179 
180 
181 private:
182  adios2::ADIOS m_ADIOS;
183 
184  struct ParameterizedOperator
185  {
186  adios2::Operator const op;
187  adios2::Params const params;
188  };
189 
190  std::vector< ParameterizedOperator > defaultOperators;
191 
192  auxiliary::TracingJSON m_config;
193  static auxiliary::TracingJSON nullvalue;
194 
195  void
196  init( nlohmann::json config );
197 
198  template< typename Key >
200  config( Key && key, auxiliary::TracingJSON & cfg )
201  {
202  if( cfg.json().is_object() && cfg.json().contains( key ) )
203  {
204  return cfg[ key ];
205  }
206  else
207  {
208  return nullvalue;
209  }
210  }
211 
212  template< typename Key >
214  config( Key && key )
215  {
216  return config< Key >( std::forward< Key >( key ), m_config );
217  }
218 
226  std::pair< std::vector< ParameterizedOperator >, bool >
227  getOperators( auxiliary::TracingJSON config );
228 
229  // use m_config
230  std::pair< std::vector< ParameterizedOperator >, bool >
231  getOperators();
232 
233  /*
234  * We need to give names to IO objects. These names are irrelevant
235  * within this application, since:
236  * 1) The name of the file written to is decided by the opened Engine's
237  * name.
238  * 2) The IOs are managed by the unordered_map m_fileData, so we do not
239  * need the ADIOS2 internal management.
240  * Since within one m_ADIOS object, the same IO name cannot be used more
241  * than once, we ensure different names by using the name counter.
242  * This allows to overwrite a file later without error.
243  */
244  int nameCounter{0};
245 
246  /*
247  * IO-heavy actions are deferred to a later point. This map stores for
248  * each open file (identified by an InvalidatableFile object) an object
249  * that manages IO-heavy actions, as well as its ADIOS2 objects, i.e.
250  * IO and Engine object.
251  * Not to be accessed directly, use getFileData().
252  */
253  std::unordered_map< InvalidatableFile,
254  std::unique_ptr< detail::BufferedActions >
255  > m_fileData;
256 
257  std::map< std::string, adios2::Operator > m_operators;
258 
259  // Overrides from AbstractIOHandlerImplCommon.
260 
261  std::string
262  filePositionToString( std::shared_ptr< ADIOS2FilePosition > ) override;
263 
264  std::shared_ptr< ADIOS2FilePosition >
265  extendFilePosition( std::shared_ptr< ADIOS2FilePosition > const & pos,
266  std::string extend ) override;
267 
268  // Helper methods.
269 
270  std::unique_ptr< adios2::Operator >
271  getCompressionOperator( std::string const & compression );
272 
273  /*
274  * The name of the ADIOS2 variable associated with this Writable.
275  * To be used for Writables that represent a dataset.
276  */
277  std::string nameOfVariable( Writable * writable );
278 
288  std::string nameOfAttribute( Writable * writable, std::string attribute );
289 
290  /*
291  * Figure out whether the Writable corresponds with a
292  * group or a dataset.
293  */
294  ADIOS2FilePosition::GD groupOrDataset( Writable * );
295 
296  detail::BufferedActions & getFileData( InvalidatableFile file );
297 
298  void dropFileData( InvalidatableFile file );
299 
300  /*
301  * Prepare a variable that already exists for an IO
302  * operation, including:
303  * (1) checking that its datatype matches T.
304  * (2) the offset and extent match the variable's shape
305  * (3) setting the offset and extent (ADIOS lingo: start
306  * and count)
307  */
308  template < typename T >
309  adios2::Variable< T > verifyDataset( Offset const & offset,
310  Extent const & extent, adios2::IO & IO,
311  std::string const & var );
312 }; // ADIOS2IOHandlerImpl
313 
314 /*
315  * The following strings are used during parsing of the JSON configuration
316  * string for the ADIOS2 backend.
317  */
318 namespace ADIOS2Defaults
319 {
320  using const_str = char const * const;
321  constexpr const_str str_engine = "engine";
322  constexpr const_str str_type = "type";
323  constexpr const_str str_params = "parameters";
324 }
325 
326 namespace detail
327 {
328  // Helper structs for calls to the switchType function
329 
331  {
333 
334 
335  explicit DatasetReader( openPMD::ADIOS2IOHandlerImpl * impl );
336 
337 
338  template < typename T >
339  void operator( )( BufferedGet & bp, adios2::IO & IO,
340  adios2::Engine & engine,
341  std::string const & fileName );
342 
343  template < int T, typename... Params > void operator( )( Params &&... );
344  };
345 
347  {
348  template < typename T >
349  Datatype operator( )( adios2::IO & IO, std::string name,
350  std::shared_ptr< Attribute::resource > resource );
351 
352  template < int n, typename... Params >
353  Datatype operator( )( Params &&... );
354  };
355 
357  {
358  template < typename T >
359  void
360  operator( )( ADIOS2IOHandlerImpl * impl, Writable * writable,
361  const Parameter< Operation::WRITE_ATT > & parameters );
362 
363 
364  template < int n, typename... Params > void operator( )( Params &&... );
365  };
366 
368  {
369  ADIOS2IOHandlerImpl * m_impl;
370 
371 
372  explicit DatasetOpener( ADIOS2IOHandlerImpl * impl );
373 
374 
375  template < typename T >
376  void operator( )( InvalidatableFile, const std::string & varName,
378 
379 
380  template < int n, typename... Params > void operator( )( Params &&... );
381  };
382 
384  {
385  ADIOS2IOHandlerImpl * m_handlerImpl;
386 
387 
388  WriteDataset( ADIOS2IOHandlerImpl * handlerImpl );
389 
390 
391  template < typename T >
392  void operator( )( BufferedPut & bp, adios2::IO & IO,
393  adios2::Engine & engine );
394 
395  template < int n, typename... Params > void operator( )( Params &&... );
396  };
397 
399  {
400  // Parameters such as DatasetHelper< T >::defineVariable
401  template < typename T, typename... Params >
402  void operator( )( Params &&... params );
403 
404  template< int n, typename... Params >
405  void
406  operator()( Params &&... );
407  };
408 
409 
410 
411  // Helper structs to help distinguish valid attribute/variable
412  // datatypes from invalid ones
413 
414 
415  /*
416  * This struct's purpose is to have specialisations
417  * for vector and array types, as well as the boolean
418  * type (which is not natively supported by ADIOS).
419  */
420  template < typename T > struct AttributeTypes
421  {
422  using Attr = adios2::Attribute< T >;
423  using BasicType = T;
424 
425  static Attr createAttribute( adios2::IO & IO, std::string name,
426  BasicType value );
427 
428  static void
429  readAttribute( adios2::IO & IO, std::string name,
430  std::shared_ptr< Attribute::resource > resource );
431  };
432 
433  template< > struct AttributeTypes< std::complex< long double > >
434  {
435  using Attr = adios2::Attribute< std::complex< double > >;
436  using BasicType = double;
437 
438  static Attr createAttribute( adios2::IO &, std::string,
439  std::complex< long double > )
440  {
441  throw std::runtime_error(
442  "[ADIOS2] Internal error: no support for long double complex attribute types" );
443  }
444 
445  static void
446  readAttribute( adios2::IO &, std::string,
447  std::shared_ptr< Attribute::resource > )
448  {
449  throw std::runtime_error(
450  "[ADIOS2] Internal error: no support for long double complex attribute types" );
451  }
452  };
453 
454  template< > struct AttributeTypes< std::vector< std::complex< long double > > >
455  {
456  using Attr = adios2::Attribute< std::complex< double > >;
457  using BasicType = double;
458 
459  static Attr createAttribute( adios2::IO &, std::string,
460  const std::vector< std::complex< long double > > & )
461  {
462  throw std::runtime_error(
463  "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
464  }
465 
466  static void
467  readAttribute( adios2::IO &, std::string,
468  std::shared_ptr< Attribute::resource > )
469  {
470  throw std::runtime_error(
471  "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
472  }
473  };
474 
475  template < typename T > struct AttributeTypes< std::vector< T > >
476  {
477  using Attr = adios2::Attribute< T >;
478  using BasicType = T;
479 
480  static Attr createAttribute( adios2::IO & IO, std::string name,
481  const std::vector< T > & value );
482 
483  static void
484  readAttribute( adios2::IO & IO, std::string name,
485  std::shared_ptr< Attribute::resource > resource );
486  };
487 
488  template < typename T, size_t n >
489  struct AttributeTypes< std::array< T, n > >
490  {
491  using Attr = adios2::Attribute< T >;
492  using BasicType = T;
493 
494  static Attr createAttribute( adios2::IO & IO, std::string name,
495  const std::array< T, n > & value );
496 
497  static void
498  readAttribute( adios2::IO & IO, std::string name,
499  std::shared_ptr< Attribute::resource > resource );
500  };
501 
502  template <> struct AttributeTypes< bool >
503  {
504  using rep = detail::bool_representation;
505  using Attr = adios2::Attribute< rep >;
506  using BasicType = rep;
507 
508  static Attr createAttribute( adios2::IO & IO, std::string name,
509  bool value );
510 
511  static void
512  readAttribute( adios2::IO & IO, std::string name,
513  std::shared_ptr< Attribute::resource > resource );
514 
515 
516  static constexpr rep toRep( bool b )
517  {
518  return b ? 1U : 0U;
519  }
520 
521 
522  static constexpr bool fromRep( rep r )
523  {
524  return r != 0;
525  }
526  };
527 
528 
534  template < typename T > struct DatasetTypes
535  {
536  static constexpr bool validType = true;
537  };
538 
539  template < typename T > struct DatasetTypes< std::vector< T > >
540  {
541  static constexpr bool validType = false;
542  };
543 
544  template <> struct DatasetTypes< bool >
545  {
546  static constexpr bool validType = false;
547  };
548 
549  // missing std::complex< long double > type in ADIOS2 v2.6.0
550  template <> struct DatasetTypes< std::complex< long double > >
551  {
552  static constexpr bool validType = false;
553  };
554 
555  template < typename T, size_t n > struct DatasetTypes< std::array< T, n > >
556  {
557  static constexpr bool validType = false;
558  };
559 
560  /*
561  * This struct's purpose is to have exactly two specialisations:
562  * (1) for types that are legal to use in a dataset
563  * (2) for types that are not legal to use in a dataset
564  * The methods in the latter specialisation will fail at runtime.
565  */
566  template < typename, typename = void > struct DatasetHelper;
567 
568  template < typename T >
570  T, typename std::enable_if< DatasetTypes< T >::validType >::type >
571  {
573 
574 
575  explicit DatasetHelper( openPMD::ADIOS2IOHandlerImpl * impl );
576 
577 
578  void openDataset( InvalidatableFile, const std::string & varName,
580 
581  void readDataset( BufferedGet &, adios2::IO &, adios2::Engine &,
582  std::string const & fileName );
583 
599  static void
600  defineVariable(
601  adios2::IO & IO,
602  std::string const & name,
603  std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const &
604  compressions,
605  adios2::Dims const & shape = adios2::Dims(),
606  adios2::Dims const & start = adios2::Dims(),
607  adios2::Dims const & count = adios2::Dims(),
608  bool const constantDims = false );
609 
610  void writeDataset( BufferedPut &, adios2::IO &, adios2::Engine & );
611  };
612 
613  template < typename T >
615  T, typename std::enable_if< !DatasetTypes< T >::validType >::type >
616  {
617  explicit DatasetHelper( openPMD::ADIOS2IOHandlerImpl * impl );
618 
619 
620  static void throwErr( );
621 
622  template < typename... Params > void openDataset( Params &&... );
623 
624  template < typename... Params > void readDataset( Params &&... );
625 
626  template < typename... Params >
627  static void defineVariable( Params &&... );
628 
629  template < typename... Params > void writeDataset( Params &&... );
630  };
631 
632  // Other datatypes used in the ADIOS2IOHandler implementation
633 
634 
635  struct BufferedActions;
636 
637  /*
638  * IO-heavy action to be executed upon flushing.
639  */
641  {
642  virtual ~BufferedAction( ) = default;
643 
644  virtual void run( BufferedActions & ) = 0;
645  };
646 
648  {
649  std::string name;
651 
652  void run( BufferedActions & ) override;
653  };
654 
656  {
657  std::string name;
659 
660  void run( BufferedActions & ) override;
661  };
662 
664  {
666  std::string name;
667 
668  void run( BufferedActions & ) override;
669  };
670 
671  /*
672  * Manages per-file information about
673  * (1) the file's IO and Engine objects
674  * (2) the file's deferred IO-heavy actions
675  */
677  {
678  BufferedActions( BufferedActions const & ) = delete;
679 
680  std::string m_file;
681  adios2::IO m_IO;
682  std::vector< std::unique_ptr< BufferedAction > > m_buffer;
688  std::unique_ptr< adios2::Engine > m_engine;
689  adios2::Mode m_mode;
690  detail::WriteDataset m_writeDataset;
691  detail::DatasetReader m_readDataset;
692  detail::AttributeReader m_attributeReader;
693  ADIOS2IOHandlerImpl & m_impl;
694 
695  using AttributeMap_t = std::map< std::string, adios2::Params >;
696 
698 
699  ~BufferedActions( );
700 
701  void
702  configure_IO( ADIOS2IOHandlerImpl & impl );
703 
704  adios2::Engine & getEngine( );
705 
706  template < typename BA > void enqueue( BA && ba );
707 
708 
709  void flush( );
710 
711  /*
712  * Delete all buffered actions without running them.
713  */
714  void drop( );
715 
716  AttributeMap_t const &
717  availableAttributes();
718 
719  std::vector< std::string >
720  availableAttributesPrefixed( std::string const & prefix );
721 
722  /*
723  * See description below.
724  */
725  void
726  invalidateAttributesMap();
727 
728  AttributeMap_t const &
729  availableVariables();
730 
731  std::vector< std::string >
732  availableVariablesPrefixed( std::string const & prefix );
733 
734  /*
735  * See description below.
736  */
737  void
738  invalidateVariablesMap();
739 
740  private:
741  /*
742  * ADIOS2 does not give direct access to its internal attribute and
743  * variable maps, but will instead give access to copies of them.
744  * In order to avoid unnecessary copies, we buffer the returned map.
745  * The downside of this is that we need to pay attention to invalidate
746  * the map whenever an attribute/variable is altered. In that case, we
747  * fetch the map anew.
748  * Revisit once https://github.com/openPMD/openPMD-api/issues/563 has
749  * been resolved
750  * If false, the buffered map has been invalidated and needs to be
751  * queried from ADIOS2 again. If true, the buffered map is equivalent to
752  * the map that would be returned by a call to
753  * IO::Available(Attributes|Variables).
754  */
755  bool m_availableAttributesValid = false;
756  AttributeMap_t m_availableAttributes;
757 
758  bool m_availableVariablesValid = false;
759  AttributeMap_t m_availableVariables;
760  };
761 
762 
763 } // namespace detail
764 #endif // openPMD_HAVE_ADIOS2
765 
766 
768 {
769 #if openPMD_HAVE_ADIOS2
770 
771 friend class ADIOS2IOHandlerImpl;
772 
773 private:
774  ADIOS2IOHandlerImpl m_impl;
775 
776 public:
777  ~ADIOS2IOHandler( ) override
778  {
779  // we must not throw in a destructor
780  try
781  {
782  this->flush( );
783  }
784  catch( std::exception const & ex )
785  {
786  std::cerr << "[~ADIOS2IOHandler] An error occurred: " << ex.what() << std::endl;
787  }
788  catch( ... )
789  {
790  std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
791  }
792  }
793 
794 #else
795 public:
796 #endif
797 
798 #if openPMD_HAVE_MPI
799 
801  std::string path,
802  Access,
803  MPI_Comm,
804  nlohmann::json options
805  );
806 
807 #endif
808 
809  ADIOS2IOHandler(std::string path, Access, nlohmann::json options );
810 
811  std::string backendName() const override { return "ADIOS2"; }
812 
813  std::future< void > flush( ) override;
814 }; // ADIOS2IOHandler
815 } // namespace openPMD
Definition: ADIOS2IOHandler.hpp:62
Wrapper around a shared pointer to:
Definition: InvalidatableFile.hpp:45
Definition: ADIOS2IOHandler.hpp:663
Definition: ADIOS2IOHandler.hpp:346
Access
File access mode to use during IO.
Definition: Access.hpp:30
Definition: ADIOS2IOHandler.hpp:398
STL namespace.
Definition: ADIOS2IOHandler.hpp:77
Definition: ADIOS2IOHandler.hpp:676
Definition: ADIOS2IOHandler.hpp:655
This struct&#39;s only field indicates whether the template parameter is a valid datatype to use for a da...
Definition: ADIOS2IOHandler.hpp:68
Definition: ADIOS2IOHandler.hpp:647
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:42
Definition: Container.cpp:51
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:68
Public definitions of openPMD-api.
Definition: Date.cpp:28
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:55
Definition: ADIOS2IOHandler.hpp:767
Definition: ADIOS2IOHandler.hpp:383
std::string backendName() const override
The currently used backend.
Definition: ADIOS2IOHandler.hpp:811
Definition: ADIOS2IOHandler.hpp:356
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON.hpp:48
Definition: Writable.hpp:43
Definition: ADIOS2IOHandler.hpp:367
Definition: ADIOS2IOHandler.hpp:330
nlohmann::json & json()
Access the underlying JSON value.
Definition: JSON.hpp:60
Definition: ADIOS2IOHandler.hpp:66
Definition: ADIOS2IOHandler.hpp:640
std::unique_ptr< adios2::Engine > m_engine
std::optional would be more idiomatic, but it&#39;s not in the C++11 standard
Definition: ADIOS2IOHandler.hpp:688