openPMD-api
HDF5IOHandlerImpl.hpp
1 /* Copyright 2017-2021 Fabian Koller
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/config.hpp"
24 #if openPMD_HAVE_HDF5
25 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
26 
27 #include "openPMD/auxiliary/JSON_internal.hpp"
28 
29 #include <hdf5.h>
30 #include <optional>
31 #include <unordered_map>
32 #include <unordered_set>
33 #endif
34 
35 namespace openPMD
36 {
37 #if openPMD_HAVE_HDF5
39 {
40 public:
43  json::TracingJSON config,
44  bool do_warn_unused_params = true);
45  ~HDF5IOHandlerImpl() override;
46 
47  void
50  void
52  void createDataset(
54  void extendDataset(
56  void availableChunks(
59  void
61  void openPath(Writable *, Parameter<Operation::OPEN_PATH> const &) override;
63  void
65  void
67  void deleteDataset(
69  void deleteAttribute(
70  Writable *, Parameter<Operation::DELETE_ATT> const &) override;
71  void
73  void writeAttribute(
74  Writable *, Parameter<Operation::WRITE_ATT> const &) override;
78  void
81  void
83 
84  std::unordered_map<Writable *, std::string> m_fileNames;
85  std::unordered_map<std::string, hid_t> m_fileNamesWithID;
86 
87  std::unordered_set<hid_t> m_openFileIDs;
88 
89  hid_t m_datasetTransferProperty;
90  hid_t m_fileAccessProperty;
91  hid_t m_fileCreateProperty;
92 
93  hbool_t m_hdf5_collective_metadata = 1;
94 
95  // h5py compatible types for bool and complex
96  hid_t m_H5T_BOOL_ENUM;
97  hid_t m_H5T_CFLOAT;
98  hid_t m_H5T_CDOUBLE;
99  hid_t m_H5T_CLONG_DOUBLE;
100  /* See https://github.com/openPMD/openPMD-api/issues/1363
101  * long double values written on AMD64 architectures cannot be read on
102  * ARM64/PPC64 architectures without doing some special tricks.
103  * We generally don't implement custom conversions, but instead pass-through
104  * the cross-platform support of HDF5.
105  * But this case is common and important enough to warrant a custom
106  * workaround.
107  */
108  hid_t m_H5T_LONG_DOUBLE_80_LE;
109  hid_t m_H5T_CLONG_DOUBLE_80_LE;
110 
111 protected:
112 #if openPMD_HAVE_MPI
113  /*
114  * Not defined in ParallelHDF5IOHandlerImpl, so we don't have to write
115  * some methods twice.
116  */
117  std::optional<MPI_Comm> m_communicator;
118 #endif
119 
120  json::TracingJSON m_config;
121  std::optional<nlohmann::json> m_buffered_dataset_config;
122 
123 private:
124  struct File
125  {
126  std::string name;
127  hid_t id;
128  };
129  std::optional<File> getFile(Writable *);
130 }; // HDF5IOHandlerImpl
131 #else
132 class HDF5IOHandlerImpl
133 {}; // HDF5IOHandlerImpl
134 #endif
135 
136 } // namespace openPMD
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:186
Definition: AbstractIOHandlerImpl.hpp:36
Definition: HDF5IOHandlerImpl.hpp:39
void createDataset(Writable *, Parameter< Operation::CREATE_DATASET > const &) override
Create a new dataset of given type, extent and storage properties.
Definition: HDF5IOHandler.cpp:441
void deleteAttribute(Writable *, Parameter< Operation::DELETE_ATT > const &) override
Delete an existing attribute.
Definition: HDF5IOHandler.cpp:1477
void listPaths(Writable *, Parameter< Operation::LIST_PATHS > &) override
List all paths/sub-groups inside a group, non-recursively.
Definition: HDF5IOHandler.cpp:2708
void openFile(Writable *, Parameter< Operation::OPEN_FILE > &) override
Open an existing file assuming it conforms to openPMD.
Definition: HDF5IOHandler.cpp:945
void closeFile(Writable *, Parameter< Operation::CLOSE_FILE > const &) override
Close the file corresponding with the writable and release file handles.
Definition: HDF5IOHandler.cpp:998
void deleteDataset(Writable *, Parameter< Operation::DELETE_DATASET > const &) override
Delete an existing dataset.
Definition: HDF5IOHandler.cpp:1425
void extendDataset(Writable *, Parameter< Operation::EXTEND_DATASET > const &) override
Increase the extent of an existing dataset.
Definition: HDF5IOHandler.cpp:811
void availableChunks(Writable *, Parameter< Operation::AVAILABLE_CHUNKS > &) override
Report chunks that are available for loading from the dataset represented by this writable.
Definition: HDF5IOHandler.cpp:874
void deregister(Writable *, Parameter< Operation::DEREGISTER > const &) override
Notify the backend that the Writable has been / will be deallocated.
Definition: HDF5IOHandler.cpp:2898
void readDataset(Writable *, Parameter< Operation::READ_DATASET > &) override
Read a chunk of data from an existing dataset.
Definition: HDF5IOHandler.cpp:1944
void writeAttribute(Writable *, Parameter< Operation::WRITE_ATT > const &) override
Create a single attribute and fill the value, possibly overwriting an existing attribute.
Definition: HDF5IOHandler.cpp:1634
void listDatasets(Writable *, Parameter< Operation::LIST_DATASETS > &) override
List all datasets inside a group, non-recursively.
Definition: HDF5IOHandler.cpp:2764
void deleteFile(Writable *, Parameter< Operation::DELETE_FILE > const &) override
Delete an existing file from physical storage.
Definition: HDF5IOHandler.cpp:1339
void createFile(Writable *, Parameter< Operation::CREATE_FILE > const &) override
Create a new file in physical storage, possibly overriding an existing file.
Definition: HDF5IOHandler.cpp:247
void checkFile(Writable *, Parameter< Operation::CHECK_FILE > &) override
Check if the file specified by the parameter is already present on disk.
Definition: HDF5IOHandler.cpp:316
void listAttributes(Writable *, Parameter< Operation::LIST_ATTS > &) override
List all attributes associated with an object.
Definition: HDF5IOHandler.cpp:2821
void openDataset(Writable *, Parameter< Operation::OPEN_DATASET > &) override
Open an existing dataset and determine its datatype and extent.
Definition: HDF5IOHandler.cpp:1103
void deletePath(Writable *, Parameter< Operation::DELETE_PATH > const &) override
Delete all objects within an existing path.
Definition: HDF5IOHandler.cpp:1373
void readAttribute(Writable *, Parameter< Operation::READ_ATT > &) override
Read the value of an existing attribute.
Definition: HDF5IOHandler.cpp:2086
void createPath(Writable *, Parameter< Operation::CREATE_PATH > const &) override
Create all necessary groups for a path, possibly recursively.
Definition: HDF5IOHandler.cpp:350
void openPath(Writable *, Parameter< Operation::OPEN_PATH > const &) override
Open all contained groups in a path, possibly recursively.
Definition: HDF5IOHandler.cpp:1016
void writeDataset(Writable *, Parameter< Operation::WRITE_DATASET > &) override
Write a chunk of data into an existing dataset.
Definition: HDF5IOHandler.cpp:1512
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:75
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON_internal.hpp:68
Public definitions of openPMD-api.
Definition: Date.cpp:29