openPMD-api
JSONIOHandlerImpl.hpp
1 /* Copyright 2017-2021 Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 
22 #pragma once
23 
24 #include "openPMD/IO/AbstractIOHandler.hpp"
25 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
26 #include "openPMD/IO/Access.hpp"
27 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
28 #include "openPMD/auxiliary/Filesystem.hpp"
29 #include "openPMD/config.hpp"
30 
31 #include <nlohmann/json.hpp>
32 
33 #include <complex>
34 #include <fstream>
35 #include <memory>
36 #include <stdexcept>
37 #include <tuple>
38 #include <unordered_map>
39 #include <unordered_set>
40 #include <vector>
41 
42 namespace openPMD
43 {
44 // Wrapper around a shared pointer to:
45 // * a filename
46 // * and a boolean indicating whether the file still exists
47 // The wrapper adds no extra information, but some commodity functions.
48 // Invariant for JSONIOHandlerImpl:
49 // For any valid filename, there is at any time at most one
50 // such shared pointer (wrapper) in the HandlerImpl's data structures
51 // (counting by pointer equality)
52 // This means, that a file can be invalidated (i.e. deleted or overwritten)
53 // by simply searching for one instance of the file e.g. in m_files and
54 // invalidating this instance
55 // A new instance may hence only be created after making sure that there are
56 // no valid instances in the data structures.
57 struct File
58 {
59  explicit File(std::string s) : fileState{std::make_shared<FileState>(s)}
60  {}
61 
62  File() = default;
63 
64  struct FileState
65  {
66  explicit FileState(std::string s) : name{std::move(s)}
67  {}
68 
69  std::string name;
70  bool valid = true;
71  };
72 
73  std::shared_ptr<FileState> fileState;
74 
75  void invalidate()
76  {
77  fileState->valid = false;
78  }
79 
80  bool valid() const
81  {
82  return fileState->valid;
83  }
84 
85  File &operator=(std::string s)
86  {
87  if (fileState)
88  {
89  fileState->name = s;
90  }
91  else
92  {
93  fileState = std::make_shared<FileState>(s);
94  }
95  return *this;
96  }
97 
98  bool operator==(File const &f) const
99  {
100  return this->fileState == f.fileState;
101  }
102 
103  std::string &operator*() const
104  {
105  return fileState->name;
106  }
107 
108  std::string *operator->() const
109  {
110  return &fileState->name;
111  }
112 
113  explicit operator bool() const
114  {
115  return fileState.operator bool();
116  }
117 };
118 } // namespace openPMD
119 
120 namespace std
121 {
122 template <>
123 struct hash<openPMD::File>
124 {
126  typedef std::size_t result_type;
127 
128  result_type operator()(argument_type const &s) const noexcept
129  {
130  return std::hash<shared_ptr<openPMD::File::FileState>>{}(s.fileState);
131  }
132 };
133 
134 // std::complex handling
135 template <class T>
136 void to_json(nlohmann::json &j, const std::complex<T> &p)
137 {
138  j = nlohmann::json{p.real(), p.imag()};
139 }
140 
141 template <class T>
142 void from_json(const nlohmann::json &j, std::complex<T> &p)
143 {
144  p.real(j.at(0));
145  p.imag(j.at(1));
146 }
147 } // namespace std
148 
149 namespace openPMD
150 {
152 {
153  using json = nlohmann::json;
154 
155 public:
157 
158  ~JSONIOHandlerImpl() override;
159 
160  void
161  createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
162 
163  void
164  createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
165 
166  void createDataset(
167  Writable *, Parameter<Operation::CREATE_DATASET> const &) override;
168 
169  void extendDataset(
170  Writable *, Parameter<Operation::EXTEND_DATASET> const &) override;
171 
172  void availableChunks(
174 
175  void openFile(Writable *, Parameter<Operation::OPEN_FILE> const &) override;
176 
177  void
178  closeFile(Writable *, Parameter<Operation::CLOSE_FILE> const &) override;
179 
180  void openPath(Writable *, Parameter<Operation::OPEN_PATH> const &) override;
181 
182  void openDataset(Writable *, Parameter<Operation::OPEN_DATASET> &) override;
183 
184  void
185  deleteFile(Writable *, Parameter<Operation::DELETE_FILE> const &) override;
186 
187  void
188  deletePath(Writable *, Parameter<Operation::DELETE_PATH> const &) override;
189 
190  void deleteDataset(
191  Writable *, Parameter<Operation::DELETE_DATASET> const &) override;
192 
193  void deleteAttribute(
194  Writable *, Parameter<Operation::DELETE_ATT> const &) override;
195 
196  void writeDataset(
197  Writable *, Parameter<Operation::WRITE_DATASET> const &) override;
198 
199  void writeAttribute(
200  Writable *, Parameter<Operation::WRITE_ATT> const &) override;
201 
202  void readDataset(Writable *, Parameter<Operation::READ_DATASET> &) override;
203 
204  void readAttribute(Writable *, Parameter<Operation::READ_ATT> &) override;
205 
206  void listPaths(Writable *, Parameter<Operation::LIST_PATHS> &) override;
207 
208  void
209  listDatasets(Writable *, Parameter<Operation::LIST_DATASETS> &) override;
210 
211  void listAttributes(Writable *, Parameter<Operation::LIST_ATTS> &) override;
212 
213  std::future<void> flush();
214 
215 private:
216  using FILEHANDLE = std::fstream;
217 
218  // map each Writable to its associated file
219  // contains only the filename, without the OS path
220  std::unordered_map<Writable *, File> m_files;
221 
222  std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
223 
224  // files that have logically, but not physically been written to
225  std::unordered_set<File> m_dirty;
226 
227  // HELPER FUNCTIONS
228 
229  // will use the IOHandler to retrieve the correct directory
230  // shared pointer to circumvent the fact that c++ pre 17 does
231  // not enforce (only allow) copy elision in return statements
232  std::shared_ptr<FILEHANDLE> getFilehandle(
233  File,
234  Access access); //, Access
235  // m_frontendAccess=this->m_handler->m_frontendAccess);
236 
237  // full operating system path of the given file
238  std::string fullPath(File);
239 
240  std::string fullPath(std::string const &);
241 
242  // from a path specification /a/b/c, remove the last
243  // "folder" (i.e. modify the string to equal /a/b)
244  static void parentDir(std::string &);
245 
246  // Fileposition is assumed to have already been set,
247  // get it in string form
248  static std::string filepositionOf(Writable *w);
249 
250  // Execute visitor on each pair of positions in the json value
251  // and the flattened multidimensional array.
252  // Used for writing from the data to JSON and for reading back into
253  // the array from JSON
254  template <typename T, typename Visitor>
255  static void syncMultidimensionalJson(
256  nlohmann::json &j,
257  Offset const &offset,
258  Extent const &extent,
259  Extent const &multiplicator,
260  Visitor visitor,
261  T *data,
262  size_t currentdim = 0);
263 
264  // multiplicators: an array [m_0,...,m_n] s.t.
265  // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n]
266  // (m_n = 1)
267  // essentially: m_i = \prod_{j=0}^{i-1} extent_j
268  static Extent getMultiplicators(Extent const &extent);
269 
270  static nlohmann::json initializeNDArray(Extent const &extent);
271 
272  static Extent getExtent(nlohmann::json &j);
273 
274  // remove single '/' in the beginning and end of a string
275  static std::string removeSlashes(std::string);
276 
277  template <typename KeyT>
278  static bool hasKey(nlohmann::json &, KeyT &&key);
279 
280  // make sure that the given path exists in proper form in
281  // the passed json value
282  static void ensurePath(nlohmann::json *json, std::string path);
283 
284  // In order not to insert the same file name into the data structures
285  // with a new pointer (e.g. when reopening), search for a possibly
286  // existing old pointer. Construct a new pointer only upon failure.
287  // The bool is true iff the pointer has been newly-created.
288  // The iterator is an iterator for m_files
289  std::tuple<File, std::unordered_map<Writable *, File>::iterator, bool>
290  getPossiblyExisting(std::string file);
291 
292  // get the json value representing the whole file, possibly reading
293  // from disk
294  std::shared_ptr<nlohmann::json> obtainJsonContents(File);
295 
296  // get the json value at the writable's fileposition
297  nlohmann::json &obtainJsonContents(Writable *writable);
298 
299  // write to disk the json contents associated with the file
300  // remove from m_dirty if unsetDirty == true
301  void putJsonContents(File, bool unsetDirty = true);
302 
303  // figure out the file position of the writable
304  // (preferring the parent's file position) and extend it
305  // by extend. return the modified file position.
306  std::shared_ptr<JSONFilePosition>
307  setAndGetFilePosition(Writable *, std::string extend);
308 
309  // figure out the file position of the writable
310  // (preferring the parent's file position)
311  // only modify the writable's fileposition when specified
312  std::shared_ptr<JSONFilePosition>
313  setAndGetFilePosition(Writable *, bool write = true);
314 
315  // get the writable's containing file
316  // if the parent is associated with another file,
317  // associate the writable with that file and return it
318  File refreshFileFromParent(Writable *writable);
319 
320  void associateWithFile(Writable *writable, File);
321 
322  // need to check the name too in order to exclude "attributes" key
323  static bool isGroup(nlohmann::json::const_iterator it);
324 
325  static bool isDataset(nlohmann::json const &j);
326 
327  // check whether the json reference contains a valid dataset
328  template <typename Param>
329  void verifyDataset(Param const &parameters, nlohmann::json &);
330 
331  static nlohmann::json platformSpecifics();
332 
333  struct DatasetWriter
334  {
335  template <typename T>
336  void operator()(
337  nlohmann::json &json,
338  const Parameter<Operation::WRITE_DATASET> &parameters);
339 
340  std::string errorMsg = "JSON: writeDataset";
341  };
342 
343  struct DatasetReader
344  {
345  template <typename T>
346  void operator()(
347  nlohmann::json &json,
349 
350  std::string errorMsg = "JSON: readDataset";
351  };
352 
353  struct AttributeWriter
354  {
355  template <typename T>
356  void operator()(nlohmann::json &, Attribute::resource const &);
357 
358  std::string errorMsg = "JSON: writeAttribute";
359  };
360 
361  struct AttributeReader
362  {
363  template <typename T>
364  void operator()(nlohmann::json &, Parameter<Operation::READ_ATT> &);
365 
366  std::string errorMsg = "JSON: writeAttribute";
367  };
368 
369  template <typename T>
370  struct CppToJSON
371  {
372  nlohmann::json operator()(T const &);
373  };
374 
375  template <typename T>
376  struct CppToJSON<std::vector<T>>
377  {
378  nlohmann::json operator()(std::vector<T> const &);
379  };
380 
381  template <typename T, int n>
382  struct CppToJSON<std::array<T, n>>
383  {
384  nlohmann::json operator()(std::array<T, n> const &);
385  };
386 
387  template <typename T, typename Enable = T>
388  struct JsonToCpp
389  {
390  T operator()(nlohmann::json const &);
391  };
392 
393  template <typename T>
394  struct JsonToCpp<std::vector<T>>
395  {
396  std::vector<T> operator()(nlohmann::json const &);
397  };
398 
399  template <typename T, int n>
400  struct JsonToCpp<std::array<T, n>>
401  {
402  std::array<T, n> operator()(nlohmann::json const &);
403  };
404 
405  template <typename T>
406  struct JsonToCpp<
407  T,
408  typename std::enable_if<std::is_floating_point<T>::value>::type>
409  {
410  T operator()(nlohmann::json const &);
411  };
412 };
413 
414 } // namespace openPMD
Definition: JSONIOHandlerImpl.hpp:64
Access
File access mode to use during IO.
Definition: Access.hpp:27
STL namespace.
Definition: JSONIOHandlerImpl.hpp:57
Definition: JSONIOHandlerImpl.hpp:151
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:122
Public definitions of openPMD-api.
Definition: Date.cpp:28
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:63
Definition: AbstractIOHandlerImpl.hpp:35