openPMD-api
JSONIOHandlerImpl.hpp
1 /* Copyright 2017-2020 Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 
22 #pragma once
23 
24 #include "openPMD/config.hpp"
25 #include "openPMD/auxiliary/Filesystem.hpp"
26 #include "openPMD/IO/AbstractIOHandler.hpp"
27 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
28 #include "openPMD/IO/Access.hpp"
29 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
30 
31 #include <nlohmann/json.hpp>
32 
33 #include <complex>
34 #include <fstream>
35 #include <memory>
36 #include <tuple>
37 #include <unordered_map>
38 #include <unordered_set>
39 #include <vector>
40 #include <stdexcept>
41 
42 
43 namespace openPMD
44 {
45  // Wrapper around a shared pointer to:
46  // * a filename
47  // * and a boolean indicating whether the file still exists
48  // The wrapper adds no extra information, but some commodity functions.
49  // Invariant for JSONIOHandlerImpl:
50  // For any valid filename, there is at any time at most one
51  // such shared pointer (wrapper) in the HandlerImpl's data structures
52  // (counting by pointer equality)
53  // This means, that a file can be invalidated (i.e. deleted or overwritten)
54  // by simply searching for one instance of the file e.g. in m_files and
55  // invalidating this instance
56  // A new instance may hence only be created after making sure that there are
57  // no valid instances in the data structures.
58  struct File
59  {
60  explicit File( std::string s ) :
61  fileState { std::make_shared< FileState >( s ) }
62  {}
63 
64 
65  File( ) = default;
66 
67 
68  struct FileState
69  {
70  explicit FileState( std::string s ) :
71  name { std::move( s ) }
72  {}
73 
74 
75  std::string name;
76  bool valid = true;
77  };
78 
79  std::shared_ptr< FileState > fileState;
80 
81 
82  void invalidate( )
83  {
84  fileState->valid = false;
85  }
86 
87 
88  bool valid( ) const
89  {
90  return fileState->valid;
91  }
92 
93 
94  File & operator=( std::string s )
95  {
96  if( fileState )
97  {
98  fileState->name = s;
99  }
100  else
101  {
102  fileState = std::make_shared< FileState >( s );
103  }
104  return *this;
105  }
106 
107 
108  bool operator==(
109  File const & f
110  ) const
111  {
112  return this->fileState == f.fileState;
113  }
114 
115 
116  std::string & operator*( ) const
117  {
118  return fileState->name;
119  }
120 
121 
122  std::string * operator->( ) const
123  {
124  return &fileState->name;
125  }
126 
127 
128  explicit operator bool( ) const
129  {
130  return fileState.operator bool( );
131  }
132  };
133 }
134 
135 namespace std
136 {
137  template< >
138  struct hash< openPMD::File >
139  {
141  typedef std::size_t result_type;
142 
143 
144  result_type operator()( argument_type const & s ) const noexcept
145  {
146  return std::hash< shared_ptr< openPMD::File::FileState>> {}( s.fileState );
147  }
148  };
149 
150  // std::complex handling
151  template< class T > void to_json(nlohmann::json &j, const std::complex< T > &p) {
152  j = nlohmann::json {p.real(), p.imag()};
153  }
154 
155  template< class T > void from_json(const nlohmann::json &j, std::complex< T > &p) {
156  p.real(j.at(0));
157  p.imag(j.at(1));
158  }
159 }
160 
161 namespace openPMD
162 {
164  public AbstractIOHandlerImpl
165  {
166  using json = nlohmann::json;
167 
168  public:
169  explicit JSONIOHandlerImpl( AbstractIOHandler * );
170 
171  ~JSONIOHandlerImpl( ) override;
172 
173  void createFile(
174  Writable *,
176  ) override;
177 
178  void createPath(
179  Writable *,
181  ) override;
182 
183  void createDataset(
184  Writable *,
186  ) override;
187 
188  void extendDataset(
189  Writable *,
191  ) override;
192 
193  void openFile(
194  Writable *,
196  ) override;
197 
198  void closeFile(
199  Writable *,
201  ) override;
202 
203  void openPath(
204  Writable *,
206  ) override;
207 
208  void openDataset(
209  Writable *,
211  ) override;
212 
213  void deleteFile(
214  Writable *,
216  ) override;
217 
218  void deletePath(
219  Writable *,
221  ) override;
222 
223  void deleteDataset(
224  Writable *,
226  ) override;
227 
228  void deleteAttribute(
229  Writable *,
231  ) override;
232 
233  void writeDataset(
234  Writable *,
236  ) override;
237 
238  void writeAttribute(
239  Writable *,
241  ) override;
242 
243  void readDataset(
244  Writable *,
246  ) override;
247 
248  void readAttribute(
249  Writable *,
251  ) override;
252 
253  void listPaths(
254  Writable *,
256  ) override;
257 
258  void listDatasets(
259  Writable *,
261  ) override;
262 
263  void listAttributes(
264  Writable *,
266  ) override;
267 
268  std::future< void > flush( ) override;
269 
270 
271  private:
272 
273  using FILEHANDLE = std::fstream;
274 
275  // map each Writable to its associated file
276  // contains only the filename, without the OS path
277  std::unordered_map<
278  Writable *,
279  File
280  > m_files;
281 
282  std::unordered_map<
283  File,
284  std::shared_ptr< nlohmann::json >> m_jsonVals;
285 
286  // files that have logically, but not physically been written to
287  std::unordered_set< File > m_dirty;
288 
289 
290  // HELPER FUNCTIONS
291 
292 
293  // will use the IOHandler to retrieve the correct directory
294  // shared pointer to circumvent the fact that c++ pre 17 does
295  // not enforce (only allow) copy elision in return statements
296  std::shared_ptr< FILEHANDLE > getFilehandle(
297  File,
298  Access access
299  ); //, Access m_frontendAccess=this->m_handler->m_frontendAccess);
300 
301  // full operating system path of the given file
302  std::string fullPath( File );
303 
304  std::string fullPath( std::string const & );
305 
306  // from a path specification /a/b/c, remove the last
307  // "folder" (i.e. modify the string to equal /a/b)
308  static void parentDir( std::string & );
309 
310  // Fileposition is assumed to have already been set,
311  // get it in string form
312  static std::string filepositionOf( Writable * w );
313 
314  // Execute visitor on each pair of positions in the json value
315  // and the flattened multidimensional array.
316  // Used for writing from the data to JSON and for reading back into
317  // the array from JSON
318  template<
319  typename T,
320  typename Visitor
321  >
322  static void syncMultidimensionalJson(
323  nlohmann::json & j,
324  Offset const & offset,
325  Extent const & extent,
326  Extent const & multiplicator,
327  Visitor visitor,
328  T * data,
329  size_t currentdim = 0
330  );
331 
332  // multiplicators: an array [m_0,...,m_n] s.t.
333  // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n]
334  // (m_n = 1)
335  // essentially: m_i = \prod_{j=0}^{i-1} extent_j
336  static Extent getMultiplicators( Extent const & extent );
337 
338  static nlohmann::json initializeNDArray( Extent const & extent );
339 
340  static Extent getExtent( nlohmann::json & j );
341 
342 
343  // remove single '/' in the beginning and end of a string
344  static std::string removeSlashes( std::string );
345 
346  template< typename KeyT >
347  static bool hasKey(
348  nlohmann::json &,
349  KeyT && key
350  );
351 
352  // make sure that the given path exists in proper form in
353  // the passed json value
354  static void ensurePath(
355  nlohmann::json * json,
356  std::string path
357  );
358 
359  // In order not to insert the same file name into the data structures
360  // with a new pointer (e.g. when reopening), search for a possibly
361  // existing old pointer. Construct a new pointer only upon failure.
362  // The bool is true iff the pointer has been newly-created.
363  // The iterator is an iterator for m_files
364  std::tuple<
365  File,
366  std::unordered_map<
367  Writable *,
368  File
369  >::iterator,
370  bool
371  > getPossiblyExisting(
372  std::string file
373  );
374 
375  // get the json value representing the whole file, possibly reading
376  // from disk
377  std::shared_ptr< nlohmann::json > obtainJsonContents( File );
378 
379  // get the json value at the writable's fileposition
380  nlohmann::json & obtainJsonContents( Writable * writable );
381 
382  // write to disk the json contents associated with the file
383  // remove from m_dirty if unsetDirty == true
384  void putJsonContents(
385  File,
386  bool unsetDirty = true
387  );
388 
389  // figure out the file position of the writable
390  // (preferring the parent's file position) and extend it
391  // by extend. return the modified file position.
392  std::shared_ptr< JSONFilePosition > setAndGetFilePosition(
393  Writable *,
394  std::string extend
395  );
396 
397  // figure out the file position of the writable
398  // (preferring the parent's file position)
399  // only modify the writable's fileposition when specified
400  std::shared_ptr< JSONFilePosition > setAndGetFilePosition(
401  Writable *,
402  bool write = true
403  );
404 
405  // get the writable's containing file
406  // if the parent is associated with another file,
407  // associate the writable with that file and return it
408  File refreshFileFromParent( Writable * writable );
409 
410  void associateWithFile(
411  Writable * writable,
412  File
413  );
414 
415  // need to check the name too in order to exclude "attributes" key
416  static bool isGroup( nlohmann::json::const_iterator it );
417 
418  static bool isDataset( nlohmann::json const & j );
419 
420 
421  // check whether the json reference contains a valid dataset
422  template< typename Param >
423  void verifyDataset(
424  Param const & parameters,
425  nlohmann::json &
426  );
427 
428  static nlohmann::json platformSpecifics( );
429 
430  struct DatasetWriter
431  {
432  template< typename T >
433  void operator()(
434  nlohmann::json & json,
435  const Parameter< Operation::WRITE_DATASET > & parameters
436  );
437 
438  template< int n >
439  void operator()(
440  nlohmann::json & json,
441  const Parameter< Operation::WRITE_DATASET > & parameters
442  );
443 
444  };
445 
446  struct DatasetReader
447  {
448  template< typename T >
449  void operator()(
450  nlohmann::json & json,
452  );
453 
454  template< int n >
455  void operator()(
456  nlohmann::json & json,
458  );
459  };
460 
461  struct AttributeWriter
462  {
463  template< typename T >
464  void operator()(
465  nlohmann::json &,
466  Attribute::resource const &
467  );
468 
469  template< int n >
470  void operator()(
471  nlohmann::json &,
472  Attribute::resource const &
473  );
474 
475  };
476 
477  struct AttributeReader
478  {
479  template< typename T >
480  void operator()(
481  nlohmann::json &,
483  );
484 
485  template< int n >
486  void operator()(
487  nlohmann::json &,
489  );
490 
491  };
492 
493  template< typename T >
494  struct CppToJSON
495  {
496  nlohmann::json operator()( T const & );
497  };
498 
499  template< typename T >
500  struct CppToJSON< std::vector< T>>
501  {
502  nlohmann::json operator()( std::vector< T > const & );
503  };
504 
505  template< typename T, int n >
506  struct CppToJSON<
507  std::array<
508  T,
509  n>>
510  {
511  nlohmann::json operator()(
512  std::array<
513  T,
514  n
515  > const &
516  );
517  };
518 
519  template<
520  typename T,
521  typename Enable = T
522  >
523  struct JsonToCpp
524  {
525  T operator()( nlohmann::json const & );
526  };
527 
528  template< typename T >
529  struct JsonToCpp< std::vector< T > >
530  {
531  std::vector< T > operator()( nlohmann::json const & );
532  };
533 
534  template< typename T, int n >
535  struct JsonToCpp<
536  std::array<
537  T,
538  n
539  >
540  >
541  {
542  std::array<
543  T,
544  n
545  > operator()( nlohmann::json const & );
546  };
547 
548  template< typename T >
549  struct JsonToCpp<
550  T,
551  typename std::enable_if<
552  std::is_floating_point<
553  T
554  >::value
555  >::type
556  >
557  {
558  T operator()( nlohmann::json const & );
559  };
560  };
561 
562 } // openPMD
Definition: JSONIOHandlerImpl.hpp:68
Access
File access mode to use during IO.
Definition: Access.hpp:30
STL namespace.
Definition: JSONIOHandlerImpl.hpp:58
Definition: JSONIOHandlerImpl.hpp:163
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:68
Public definitions of openPMD-api.
Definition: Date.cpp:28
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:55
Definition: AbstractIOHandlerImpl.hpp:35