openPMD-api
JSONIOHandlerImpl.hpp
1 /* Copyright 2017-2021 Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 
22 #pragma once
23 
24 #include "openPMD/IO/AbstractIOHandler.hpp"
25 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
26 #include "openPMD/IO/Access.hpp"
27 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
28 #include "openPMD/auxiliary/Filesystem.hpp"
29 #include "openPMD/config.hpp"
30 
31 #include <nlohmann/json.hpp>
32 
33 #include <complex>
34 #include <fstream>
35 #include <memory>
36 #include <stdexcept>
37 #include <tuple>
38 #include <unordered_map>
39 #include <unordered_set>
40 #include <vector>
41 
42 namespace openPMD
43 {
44 // Wrapper around a shared pointer to:
45 // * a filename
46 // * and a boolean indicating whether the file still exists
47 // The wrapper adds no extra information, but some commodity functions.
48 // Invariant for JSONIOHandlerImpl:
49 // For any valid filename, there is at any time at most one
50 // such shared pointer (wrapper) in the HandlerImpl's data structures
51 // (counting by pointer equality)
52 // This means, that a file can be invalidated (i.e. deleted or overwritten)
53 // by simply searching for one instance of the file e.g. in m_files and
54 // invalidating this instance
55 // A new instance may hence only be created after making sure that there are
56 // no valid instances in the data structures.
57 struct File
58 {
59  explicit File(std::string s) : fileState{std::make_shared<FileState>(s)}
60  {}
61 
62  File() = default;
63 
64  struct FileState
65  {
66  explicit FileState(std::string s) : name{std::move(s)}
67  {}
68 
69  std::string name;
70  bool valid = true;
71  };
72 
73  std::shared_ptr<FileState> fileState;
74 
75  void invalidate()
76  {
77  fileState->valid = false;
78  }
79 
80  bool valid() const
81  {
82  return fileState->valid;
83  }
84 
85  File &operator=(std::string s)
86  {
87  if (fileState)
88  {
89  fileState->name = s;
90  }
91  else
92  {
93  fileState = std::make_shared<FileState>(s);
94  }
95  return *this;
96  }
97 
98  bool operator==(File const &f) const
99  {
100  return this->fileState == f.fileState;
101  }
102 
103  std::string &operator*() const
104  {
105  return fileState->name;
106  }
107 
108  std::string *operator->() const
109  {
110  return &fileState->name;
111  }
112 
113  explicit operator bool() const
114  {
115  return fileState.operator bool();
116  }
117 };
118 } // namespace openPMD
119 
120 namespace std
121 {
122 template <>
123 struct hash<openPMD::File>
124 {
126  typedef std::size_t result_type;
127 
128  result_type operator()(argument_type const &s) const noexcept
129  {
130  return std::hash<shared_ptr<openPMD::File::FileState>>{}(s.fileState);
131  }
132 };
133 
134 // std::complex handling
135 template <class T>
136 void to_json(nlohmann::json &j, const std::complex<T> &p)
137 {
138  j = nlohmann::json{p.real(), p.imag()};
139 }
140 
141 template <class T>
142 void from_json(const nlohmann::json &j, std::complex<T> &p)
143 {
144  p.real(j.at(0));
145  p.imag(j.at(1));
146 }
147 } // namespace std
148 
149 namespace openPMD
150 {
152 {
153  using json = nlohmann::json;
154 
155 public:
157 
158  ~JSONIOHandlerImpl() override;
159 
160  void
161  createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
162 
163  void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) override;
164 
165  void
166  createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
167 
168  void createDataset(
169  Writable *, Parameter<Operation::CREATE_DATASET> const &) override;
170 
171  void extendDataset(
172  Writable *, Parameter<Operation::EXTEND_DATASET> const &) override;
173 
174  void availableChunks(
176 
177  void openFile(Writable *, Parameter<Operation::OPEN_FILE> &) override;
178 
179  void
180  closeFile(Writable *, Parameter<Operation::CLOSE_FILE> const &) override;
181 
182  void openPath(Writable *, Parameter<Operation::OPEN_PATH> const &) override;
183 
184  void openDataset(Writable *, Parameter<Operation::OPEN_DATASET> &) override;
185 
186  void
187  deleteFile(Writable *, Parameter<Operation::DELETE_FILE> const &) override;
188 
189  void
190  deletePath(Writable *, Parameter<Operation::DELETE_PATH> const &) override;
191 
192  void deleteDataset(
193  Writable *, Parameter<Operation::DELETE_DATASET> const &) override;
194 
195  void deleteAttribute(
196  Writable *, Parameter<Operation::DELETE_ATT> const &) override;
197 
198  void
199  writeDataset(Writable *, Parameter<Operation::WRITE_DATASET> &) override;
200 
201  void writeAttribute(
202  Writable *, Parameter<Operation::WRITE_ATT> const &) override;
203 
204  void readDataset(Writable *, Parameter<Operation::READ_DATASET> &) override;
205 
206  void readAttribute(Writable *, Parameter<Operation::READ_ATT> &) override;
207 
208  void listPaths(Writable *, Parameter<Operation::LIST_PATHS> &) override;
209 
210  void
211  listDatasets(Writable *, Parameter<Operation::LIST_DATASETS> &) override;
212 
213  void listAttributes(Writable *, Parameter<Operation::LIST_ATTS> &) override;
214 
215  void
216  deregister(Writable *, Parameter<Operation::DEREGISTER> const &) override;
217 
218  std::future<void> flush();
219 
220 private:
221  using FILEHANDLE = std::fstream;
222 
223  // map each Writable to its associated file
224  // contains only the filename, without the OS path
225  std::unordered_map<Writable *, File> m_files;
226 
227  std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
228 
229  // files that have logically, but not physically been written to
230  std::unordered_set<File> m_dirty;
231 
232  // HELPER FUNCTIONS
233 
234  // will use the IOHandler to retrieve the correct directory
235  // shared pointer to circumvent the fact that c++ pre 17 does
236  // not enforce (only allow) copy elision in return statements
237  std::shared_ptr<FILEHANDLE> getFilehandle(
238  File,
239  Access access); //, Access
240  // m_frontendAccess=this->m_handler->m_frontendAccess);
241 
242  // full operating system path of the given file
243  std::string fullPath(File);
244 
245  std::string fullPath(std::string const &);
246 
247  // from a path specification /a/b/c, remove the last
248  // "folder" (i.e. modify the string to equal /a/b)
249  static void parentDir(std::string &);
250 
251  // Fileposition is assumed to have already been set,
252  // get it in string form
253  static std::string filepositionOf(Writable *w);
254 
255  // Execute visitor on each pair of positions in the json value
256  // and the flattened multidimensional array.
257  // Used for writing from the data to JSON and for reading back into
258  // the array from JSON
259  template <typename T, typename Visitor>
260  static void syncMultidimensionalJson(
261  nlohmann::json &j,
262  Offset const &offset,
263  Extent const &extent,
264  Extent const &multiplicator,
265  Visitor visitor,
266  T *data,
267  size_t currentdim = 0);
268 
269  // multiplicators: an array [m_0,...,m_n] s.t.
270  // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n]
271  // (m_n = 1)
272  // essentially: m_i = \prod_{j=0}^{i-1} extent_j
273  static Extent getMultiplicators(Extent const &extent);
274 
275  static nlohmann::json initializeNDArray(Extent const &extent);
276 
277  static Extent getExtent(nlohmann::json &j);
278 
279  // remove single '/' in the beginning and end of a string
280  static std::string removeSlashes(std::string);
281 
282  template <typename KeyT>
283  static bool hasKey(nlohmann::json &, KeyT &&key);
284 
285  // make sure that the given path exists in proper form in
286  // the passed json value
287  static void ensurePath(nlohmann::json *json, std::string path);
288 
289  // In order not to insert the same file name into the data structures
290  // with a new pointer (e.g. when reopening), search for a possibly
291  // existing old pointer. Construct a new pointer only upon failure.
292  // The bool is true iff the pointer has been newly-created.
293  // The iterator is an iterator for m_files
294  std::tuple<File, std::unordered_map<Writable *, File>::iterator, bool>
295  getPossiblyExisting(std::string file);
296 
297  // get the json value representing the whole file, possibly reading
298  // from disk
299  std::shared_ptr<nlohmann::json> obtainJsonContents(File);
300 
301  // get the json value at the writable's fileposition
302  nlohmann::json &obtainJsonContents(Writable *writable);
303 
304  // write to disk the json contents associated with the file
305  // remove from m_dirty if unsetDirty == true
306  void putJsonContents(File, bool unsetDirty = true);
307 
308  // figure out the file position of the writable
309  // (preferring the parent's file position) and extend it
310  // by extend. return the modified file position.
311  std::shared_ptr<JSONFilePosition>
312  setAndGetFilePosition(Writable *, std::string extend);
313 
314  // figure out the file position of the writable
315  // (preferring the parent's file position)
316  // only modify the writable's fileposition when specified
317  std::shared_ptr<JSONFilePosition>
318  setAndGetFilePosition(Writable *, bool write = true);
319 
320  // get the writable's containing file
321  // if the parent is associated with another file,
322  // associate the writable with that file and return it
323  File refreshFileFromParent(Writable *writable);
324 
325  void associateWithFile(Writable *writable, File);
326 
327  // need to check the name too in order to exclude "attributes" key
328  static bool isGroup(nlohmann::json::const_iterator it);
329 
330  static bool isDataset(nlohmann::json const &j);
331 
332  // check whether the json reference contains a valid dataset
333  template <typename Param>
334  void verifyDataset(Param const &parameters, nlohmann::json &);
335 
336  static nlohmann::json platformSpecifics();
337 
338  struct DatasetWriter
339  {
340  template <typename T>
341  static void call(
342  nlohmann::json &json,
343  const Parameter<Operation::WRITE_DATASET> &parameters);
344 
345  static constexpr char const *errorMsg = "JSON: writeDataset";
346  };
347 
348  struct DatasetReader
349  {
350  template <typename T>
351  static void call(
352  nlohmann::json &json,
354 
355  static constexpr char const *errorMsg = "JSON: readDataset";
356  };
357 
358  struct AttributeWriter
359  {
360  template <typename T>
361  static void call(nlohmann::json &, Attribute::resource const &);
362 
363  static constexpr char const *errorMsg = "JSON: writeAttribute";
364  };
365 
366  struct AttributeReader
367  {
368  template <typename T>
369  static void call(nlohmann::json &, Parameter<Operation::READ_ATT> &);
370 
371  static constexpr char const *errorMsg = "JSON: writeAttribute";
372  };
373 
374  template <typename T>
375  struct CppToJSON
376  {
377  nlohmann::json operator()(T const &);
378  };
379 
380  template <typename T>
381  struct CppToJSON<std::vector<T>>
382  {
383  nlohmann::json operator()(std::vector<T> const &);
384  };
385 
386  template <typename T, int n>
387  struct CppToJSON<std::array<T, n>>
388  {
389  nlohmann::json operator()(std::array<T, n> const &);
390  };
391 
392  template <typename T, typename Enable = T>
393  struct JsonToCpp
394  {
395  T operator()(nlohmann::json const &);
396  };
397 
398  template <typename T>
399  struct JsonToCpp<std::vector<T>>
400  {
401  std::vector<T> operator()(nlohmann::json const &);
402  };
403 
404  template <typename T, int n>
405  struct JsonToCpp<std::array<T, n>>
406  {
407  std::array<T, n> operator()(nlohmann::json const &);
408  };
409 
410  template <typename T>
411  struct JsonToCpp<
412  T,
413  typename std::enable_if<std::is_floating_point<T>::value>::type>
414  {
415  T operator()(nlohmann::json const &);
416  };
417 };
418 
419 } // namespace openPMD
Definition: JSONIOHandlerImpl.hpp:64
Access
File access mode to use during IO.
Definition: Access.hpp:29
STL namespace.
Definition: JSONIOHandlerImpl.hpp:57
Definition: JSONIOHandlerImpl.hpp:151
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:179
Public definitions of openPMD-api.
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:64
Definition: AbstractIOHandlerImpl.hpp:35