openPMD-api
JSONIOHandlerImpl.hpp
1 /* Copyright 2017-2021 Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 
22 #pragma once
23 
24 #include "openPMD/IO/AbstractIOHandler.hpp"
25 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
26 #include "openPMD/IO/Access.hpp"
27 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
28 #include "openPMD/auxiliary/Filesystem.hpp"
29 #include "openPMD/auxiliary/JSON_internal.hpp"
30 #include "openPMD/config.hpp"
31 
32 #include <istream>
33 #include <nlohmann/json.hpp>
34 #if openPMD_HAVE_MPI
35 #include <mpi.h>
36 #endif
37 
38 #include <complex>
39 #include <fstream>
40 #include <memory>
41 #include <stdexcept>
42 #include <tuple>
43 #include <unordered_map>
44 #include <unordered_set>
45 #include <vector>
46 
47 namespace openPMD
48 {
49 // Wrapper around a shared pointer to:
50 // * a filename
51 // * and a boolean indicating whether the file still exists
52 // The wrapper adds no extra information, but some commodity functions.
53 // Invariant for JSONIOHandlerImpl:
54 // For any valid filename, there is at any time at most one
55 // such shared pointer (wrapper) in the HandlerImpl's data structures
56 // (counting by pointer equality)
57 // This means, that a file can be invalidated (i.e. deleted or overwritten)
58 // by simply searching for one instance of the file e.g. in m_files and
59 // invalidating this instance
60 // A new instance may hence only be created after making sure that there are
61 // no valid instances in the data structures.
62 struct File
63 {
64  explicit File(std::string s) : fileState{std::make_shared<FileState>(s)}
65  {}
66 
67  File() = default;
68 
69  struct FileState
70  {
71  explicit FileState(std::string s) : name{std::move(s)}
72  {}
73 
74  std::string name;
75  bool valid = true;
76  bool printedReadmeWarningAlready = false;
77  };
78 
79  std::shared_ptr<FileState> fileState;
80 
81  void invalidate()
82  {
83  fileState->valid = false;
84  }
85 
86  bool valid() const
87  {
88  return fileState->valid;
89  }
90 
91  File &operator=(std::string const &s)
92  {
93  if (fileState)
94  {
95  fileState->name = s;
96  }
97  else
98  {
99  fileState = std::make_shared<FileState>(s);
100  }
101  return *this;
102  }
103 
104  bool operator==(File const &f) const
105  {
106  return this->fileState == f.fileState;
107  }
108 
109  std::string &operator*() const
110  {
111  return fileState->name;
112  }
113 
114  std::string *operator->() const
115  {
116  return &fileState->name;
117  }
118 
119  explicit operator bool() const
120  {
121  return fileState.operator bool();
122  }
123 };
124 } // namespace openPMD
125 
126 namespace std
127 {
128 template <>
129 struct hash<openPMD::File>
130 {
132  typedef std::size_t result_type;
133 
134  result_type operator()(argument_type const &s) const noexcept
135  {
136  return std::hash<shared_ptr<openPMD::File::FileState>>{}(s.fileState);
137  }
138 };
139 
140 // std::complex handling
141 template <class T>
142 void to_json(nlohmann::json &j, const std::complex<T> &p)
143 {
144  j = nlohmann::json{p.real(), p.imag()};
145 }
146 
147 template <class T>
148 void from_json(const nlohmann::json &j, std::complex<T> &p)
149 {
150  p.real(j.at(0));
151  p.imag(j.at(1));
152 }
153 } // namespace std
154 
155 namespace openPMD
156 {
158 {
159  using json = nlohmann::json;
160 
161 public:
162  enum class FileFormat
163  {
164  Json,
165  Toml
166  };
167 
168  explicit JSONIOHandlerImpl(
171  FileFormat,
172  std::string originalExtension);
173 
174 #if openPMD_HAVE_MPI
177  MPI_Comm,
179  FileFormat,
180  std::string originalExtension);
181 #endif
182 
183  ~JSONIOHandlerImpl() override;
184 
185  void
187 
189 
190  void
192 
193  void createDataset(
194  Writable *, Parameter<Operation::CREATE_DATASET> const &) override;
195 
196  void extendDataset(
197  Writable *, Parameter<Operation::EXTEND_DATASET> const &) override;
198 
199  void availableChunks(
201 
203 
204  void
206 
207  void openPath(Writable *, Parameter<Operation::OPEN_PATH> const &) override;
208 
210 
211  void
213 
214  void
216 
217  void deleteDataset(
218  Writable *, Parameter<Operation::DELETE_DATASET> const &) override;
219 
220  void deleteAttribute(
221  Writable *, Parameter<Operation::DELETE_ATT> const &) override;
222 
223  void
225 
226  void writeAttribute(
227  Writable *, Parameter<Operation::WRITE_ATT> const &) override;
228 
230 
232 
234 
235  void
237 
239 
240  void
242 
243  std::future<void> flush();
244 
245 private:
246 #if openPMD_HAVE_MPI
247  std::optional<MPI_Comm> m_communicator;
248 #endif
249 
250  using FILEHANDLE = std::fstream;
251 
252  // map each Writable to its associated file
253  // contains only the filename, without the OS path
254  std::unordered_map<Writable *, File> m_files;
255 
256  std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
257 
258  // files that have logically, but not physically been written to
259  std::unordered_set<File> m_dirty;
260 
261  /*
262  * Is set by constructor.
263  */
264  FileFormat m_fileFormat{};
265 
266  std::string m_originalExtension;
267 
268  // HELPER FUNCTIONS
269 
270  // will use the IOHandler to retrieve the correct directory.
271  // first tuple element will be the underlying opened file handle.
272  // if Access is read mode, then the second tuple element will be the istream
273  // casted to precision std::numeric_limits<double>::digits10 + 1, else null.
274  // if Access is write mode, then the second tuple element will be the
275  // ostream casted to precision std::numeric_limits<double>::digits10 + 1,
276  // else null. first tuple element needs to be a pointer, since the casted
277  // streams are references only.
278  std::tuple<std::unique_ptr<FILEHANDLE>, std::istream *, std::ostream *>
279  getFilehandle(File const &, Access access);
280 
281  // full operating system path of the given file
282  std::string fullPath(File const &);
283 
284  std::string fullPath(std::string const &);
285 
286  // from a path specification /a/b/c, remove the last
287  // "folder" (i.e. modify the string to equal /a/b)
288  static void parentDir(std::string &);
289 
290  // Fileposition is assumed to have already been set,
291  // get it in string form
292  static std::string filepositionOf(Writable *w);
293 
294  // Execute visitor on each pair of positions in the json value
295  // and the flattened multidimensional array.
296  // Used for writing from the data to JSON and for reading back into
297  // the array from JSON
298  template <typename T, typename Visitor>
299  static void syncMultidimensionalJson(
300  nlohmann::json &j,
301  Offset const &offset,
302  Extent const &extent,
303  Extent const &multiplicator,
304  Visitor visitor,
305  T *data,
306  size_t currentdim = 0);
307 
308  // multiplicators: an array [m_0,...,m_n] s.t.
309  // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n]
310  // (m_n = 1)
311  // essentially: m_i = \prod_{j=0}^{i-1} extent_j
312  static Extent getMultiplicators(Extent const &extent);
313 
314  static Extent getExtent(nlohmann::json &j);
315 
316  // remove single '/' in the beginning and end of a string
317  static std::string removeSlashes(std::string);
318 
319  template <typename KeyT>
320  static bool hasKey(nlohmann::json const &, KeyT &&key);
321 
322  // make sure that the given path exists in proper form in
323  // the passed json value
324  static void ensurePath(nlohmann::json *json, std::string const &path);
325 
326  // In order not to insert the same file name into the data structures
327  // with a new pointer (e.g. when reopening), search for a possibly
328  // existing old pointer. Construct a new pointer only upon failure.
329  // The bool is true iff the pointer has been newly-created.
330  // The iterator is an iterator for m_files
331  std::tuple<File, std::unordered_map<Writable *, File>::iterator, bool>
332  getPossiblyExisting(std::string const &file);
333 
334  // get the json value representing the whole file, possibly reading
335  // from disk
336  std::shared_ptr<nlohmann::json> obtainJsonContents(File const &);
337 
338  // get the json value at the writable's fileposition
339  nlohmann::json &obtainJsonContents(Writable *writable);
340 
341  // write to disk the json contents associated with the file
342  // remove from m_dirty if unsetDirty == true
343  auto putJsonContents(File const &, bool unsetDirty = true)
344  -> decltype(m_jsonVals)::iterator;
345 
346  // figure out the file position of the writable
347  // (preferring the parent's file position) and extend it
348  // by extend. return the modified file position.
349  std::shared_ptr<JSONFilePosition>
350  setAndGetFilePosition(Writable *, std::string const &extend);
351 
352  // figure out the file position of the writable
353  // (preferring the parent's file position)
354  // only modify the writable's fileposition when specified
355  std::shared_ptr<JSONFilePosition>
356  setAndGetFilePosition(Writable *, bool write = true);
357 
358  // get the writable's containing file
359  // if the parent is associated with another file,
360  // associate the writable with that file and return it
361  File refreshFileFromParent(Writable *writable);
362 
363  void associateWithFile(Writable *writable, File);
364 
365  // need to check the name too in order to exclude "attributes" key
366  static bool isGroup(nlohmann::json::const_iterator const &it);
367 
368  static bool isDataset(nlohmann::json const &j);
369 
370  // check whether the json reference contains a valid dataset
371  template <typename Param>
372  void verifyDataset(Param const &parameters, nlohmann::json &);
373 
374  static nlohmann::json platformSpecifics();
375 
376  struct DatasetWriter
377  {
378  template <typename T>
379  static void call(
380  nlohmann::json &json,
381  const Parameter<Operation::WRITE_DATASET> &parameters);
382 
383  static constexpr char const *errorMsg = "JSON: writeDataset";
384  };
385 
386  struct DatasetReader
387  {
388  template <typename T>
389  static void call(
390  nlohmann::json &json,
392 
393  static constexpr char const *errorMsg = "JSON: readDataset";
394  };
395 
396  struct AttributeWriter
397  {
398  template <typename T>
399  static void call(nlohmann::json &, Attribute::resource const &);
400 
401  static constexpr char const *errorMsg = "JSON: writeAttribute";
402  };
403 
404  struct AttributeReader
405  {
406  template <typename T>
407  static void
408  call(nlohmann::json const &, Parameter<Operation::READ_ATT> &);
409 
410  static constexpr char const *errorMsg = "JSON: writeAttribute";
411  };
412 
413  template <typename T>
414  struct CppToJSON
415  {
416  nlohmann::json operator()(T const &);
417  };
418 
419  template <typename T>
420  struct CppToJSON<std::vector<T>>
421  {
422  nlohmann::json operator()(std::vector<T> const &);
423  };
424 
425  template <typename T, int n>
426  struct CppToJSON<std::array<T, n>>
427  {
428  nlohmann::json operator()(std::array<T, n> const &);
429  };
430 
431  template <typename T, typename Enable = T>
432  struct JsonToCpp
433  {
434  T operator()(nlohmann::json const &);
435  };
436 
437  template <typename T>
438  struct JsonToCpp<std::vector<T>>
439  {
440  std::vector<T> operator()(nlohmann::json const &);
441  };
442 
443  template <typename T, int n>
444  struct JsonToCpp<std::array<T, n>>
445  {
446  std::array<T, n> operator()(nlohmann::json const &);
447  };
448 
449  template <typename T>
450  struct JsonToCpp<
451  T,
452  typename std::enable_if<std::is_floating_point<T>::value>::type>
453  {
454  T operator()(nlohmann::json const &);
455  };
456 };
457 
458 } // namespace openPMD
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:186
Definition: AbstractIOHandlerImpl.hpp:36
Definition: JSONIOHandlerImpl.hpp:158
void createPath(Writable *, Parameter< Operation::CREATE_PATH > const &) override
Create all necessary groups for a path, possibly recursively.
Definition: JSONIOHandlerImpl.cpp:239
void availableChunks(Writable *, Parameter< Operation::AVAILABLE_CHUNKS > &) override
Report chunks that are available for loading from the dataset represented by this writable.
Definition: JSONIOHandlerImpl.cpp:603
void listAttributes(Writable *, Parameter< Operation::LIST_ATTS > &) override
List all attributes associated with an object.
Definition: JSONIOHandlerImpl.cpp:1016
void checkFile(Writable *, Parameter< Operation::CHECK_FILE > &) override
Check if the file specified by the parameter is already present on disk.
Definition: JSONIOHandlerImpl.cpp:223
void openPath(Writable *, Parameter< Operation::OPEN_PATH > const &) override
Open all contained groups in a path, possibly recursively.
Definition: JSONIOHandlerImpl.cpp:653
void readDataset(Writable *, Parameter< Operation::READ_DATASET > &) override
Read a chunk of data from an existing dataset.
Definition: JSONIOHandlerImpl.cpp:921
void deleteAttribute(Writable *, Parameter< Operation::DELETE_ATT > const &) override
Delete an existing attribute.
Definition: JSONIOHandlerImpl.cpp:851
void openFile(Writable *, Parameter< Operation::OPEN_FILE > &) override
Open an existing file assuming it conforms to openPMD.
Definition: JSONIOHandlerImpl.cpp:613
void openDataset(Writable *, Parameter< Operation::OPEN_DATASET > &) override
Open an existing dataset and determine its datatype and extent.
Definition: JSONIOHandlerImpl.cpp:679
void deletePath(Writable *, Parameter< Operation::DELETE_PATH > const &) override
Delete all objects within an existing path.
Definition: JSONIOHandlerImpl.cpp:731
void writeAttribute(Writable *, Parameter< Operation::WRITE_ATT > const &) override
Create a single attribute and fill the value, possibly overwriting an existing attribute.
Definition: JSONIOHandlerImpl.cpp:887
void deregister(Writable *, Parameter< Operation::DEREGISTER > const &) override
Notify the backend that the Writable has been / will be deallocated.
Definition: JSONIOHandlerImpl.cpp:1036
void readAttribute(Writable *, Parameter< Operation::READ_ATT > &) override
Read the value of an existing attribute.
Definition: JSONIOHandlerImpl.cpp:940
void createFile(Writable *, Parameter< Operation::CREATE_FILE > const &) override
Create a new file in physical storage, possibly overriding an existing file.
Definition: JSONIOHandlerImpl.cpp:169
void writeDataset(Writable *, Parameter< Operation::WRITE_DATASET > &) override
Write a chunk of data into an existing dataset.
Definition: JSONIOHandlerImpl.cpp:868
void deleteFile(Writable *, Parameter< Operation::DELETE_FILE > const &) override
Delete an existing file from physical storage.
Definition: JSONIOHandlerImpl.cpp:700
void listPaths(Writable *, Parameter< Operation::LIST_PATHS > &) override
List all paths/sub-groups inside a group, non-recursively.
Definition: JSONIOHandlerImpl.cpp:979
void closeFile(Writable *, Parameter< Operation::CLOSE_FILE > const &) override
Close the file corresponding with the writable and release file handles.
Definition: JSONIOHandlerImpl.cpp:635
void listDatasets(Writable *, Parameter< Operation::LIST_DATASETS > &) override
List all datasets inside a group, non-recursively.
Definition: JSONIOHandlerImpl.cpp:998
void createDataset(Writable *, Parameter< Operation::CREATE_DATASET > const &) override
Create a new dataset of given type, extent and storage properties.
Definition: JSONIOHandlerImpl.cpp:274
void deleteDataset(Writable *, Parameter< Operation::DELETE_DATASET > const &) override
Delete an existing dataset.
Definition: JSONIOHandlerImpl.cpp:809
void extendDataset(Writable *, Parameter< Operation::EXTEND_DATASET > const &) override
Increase the extent of an existing dataset.
Definition: JSONIOHandlerImpl.cpp:350
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:75
Extend nlohmann::json with tracing of which keys have been accessed by operator[]().
Definition: JSON_internal.hpp:68
Public definitions of openPMD-api.
Definition: Date.cpp:29
Access
File access mode to use during IO.
Definition: Access.hpp:30
Definition: JSONIOHandlerImpl.hpp:70
Definition: JSONIOHandlerImpl.hpp:63