openPMD-api
JSONIOHandlerImpl.hpp
1 /* Copyright 2017-2020 Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 
22 #pragma once
23 
24 #include "openPMD/config.hpp"
25 #include "openPMD/auxiliary/Filesystem.hpp"
26 #include "openPMD/IO/AbstractIOHandler.hpp"
27 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
28 #include "openPMD/IO/AccessType.hpp"
29 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
30 
31 #include <nlohmann/json.hpp>
32 
33 #include <fstream>
34 #include <memory>
35 #include <tuple>
36 #include <unordered_map>
37 #include <unordered_set>
38 #include <vector>
39 #include <stdexcept>
40 
41 
42 namespace openPMD
43 {
44  // Wrapper around a shared pointer to:
45  // * a filename
46  // * and a boolean indicating whether the file still exists
47  // The wrapper adds no extra information, but some commodity functions.
48  // Invariant for JSONIOHandlerImpl:
49  // For any valid filename, there is at any time at most one
50  // such shared pointer (wrapper) in the HandlerImpl's data structures
51  // (counting by pointer equality)
52  // This means, that a file can be invalidated (i.e. deleted or overwritten)
53  // by simply searching for one instance of the file e.g. in m_files and
54  // invalidating this instance
55  // A new instance may hence only be created after making sure that there are
56  // no valid instances in the data structures.
57  struct File
58  {
59  explicit File( std::string s ) :
60  fileState { std::make_shared< FileState >( s ) }
61  {}
62 
63 
64  File( ) = default;
65 
66 
67  struct FileState
68  {
69  explicit FileState( std::string s ) :
70  name { std::move( s ) }
71  {}
72 
73 
74  std::string name;
75  bool valid = true;
76  };
77 
78  std::shared_ptr< FileState > fileState;
79 
80 
81  void invalidate( )
82  {
83  fileState->valid = false;
84  }
85 
86 
87  bool valid( ) const
88  {
89  return fileState->valid;
90  }
91 
92 
93  File & operator=( std::string s )
94  {
95  if( fileState )
96  {
97  fileState->name = s;
98  }
99  else
100  {
101  fileState = std::make_shared< FileState >( s );
102  }
103  return *this;
104  }
105 
106 
107  bool operator==(
108  File const & f
109  ) const
110  {
111  return this->fileState == f.fileState;
112  }
113 
114 
115  std::string & operator*( ) const
116  {
117  return fileState->name;
118  }
119 
120 
121  std::string * operator->( ) const
122  {
123  return &fileState->name;
124  }
125 
126 
127  explicit operator bool( ) const
128  {
129  return fileState.operator bool( );
130  }
131  };
132 }
133 
134 namespace std
135 {
136  template< >
137  struct hash< openPMD::File >
138  {
140  typedef std::size_t result_type;
141 
142 
143  result_type operator()( argument_type const & s ) const noexcept
144  {
145  return std::hash< shared_ptr< openPMD::File::FileState>> {}( s.fileState );
146  }
147  };
148 }
149 
150 namespace openPMD
151 {
153  public AbstractIOHandlerImpl
154  {
155  using json = nlohmann::json;
156 
157  public:
158  explicit JSONIOHandlerImpl( AbstractIOHandler * );
159 
160  ~JSONIOHandlerImpl( ) override;
161 
162  void createFile(
163  Writable *,
165  ) override;
166 
167  void createPath(
168  Writable *,
170  ) override;
171 
172  void createDataset(
173  Writable *,
175  ) override;
176 
177  void extendDataset(
178  Writable *,
180  ) override;
181 
182  void openFile(
183  Writable *,
185  ) override;
186 
187  void openPath(
188  Writable *,
190  ) override;
191 
192  void openDataset(
193  Writable *,
195  ) override;
196 
197  void deleteFile(
198  Writable *,
200  ) override;
201 
202  void deletePath(
203  Writable *,
205  ) override;
206 
207  void deleteDataset(
208  Writable *,
210  ) override;
211 
212  void deleteAttribute(
213  Writable *,
215  ) override;
216 
217  void writeDataset(
218  Writable *,
220  ) override;
221 
222  void writeAttribute(
223  Writable *,
225  ) override;
226 
227  void readDataset(
228  Writable *,
230  ) override;
231 
232  void readAttribute(
233  Writable *,
235  ) override;
236 
237  void listPaths(
238  Writable *,
240  ) override;
241 
242  void listDatasets(
243  Writable *,
245  ) override;
246 
247  void listAttributes(
248  Writable *,
250  ) override;
251 
252  std::future< void > flush( ) override;
253 
254 
255  private:
256 
257  using FILEHANDLE = std::fstream;
258 
259  // map each Writable to its associated file
260  // contains only the filename, without the OS path
261  std::unordered_map<
262  Writable *,
263  File
264  > m_files;
265 
266  std::unordered_map<
267  File,
268  std::shared_ptr< nlohmann::json >> m_jsonVals;
269 
270  // files that have logically, but not physically been written to
271  std::unordered_set< File > m_dirty;
272 
273 
274  // HELPER FUNCTIONS
275 
276 
277  // will use the IOHandler to retrieve the correct directory
278  // shared pointer to circumvent the fact that c++ pre 17 does
279  // not enforce (only allow) copy elision in return statements
280  std::shared_ptr< FILEHANDLE > getFilehandle(
281  File,
282  AccessType accessType
283  ); //, AccessType accessTypeFrontend=this->m_handler->accessTypeFrontend);
284 
285  // full operating system path of the given file
286  std::string fullPath( File );
287 
288  std::string fullPath( std::string const & );
289 
290  // from a path specification /a/b/c, remove the last
291  // "folder" (i.e. modify the string to equal /a/b)
292  static void parentDir( std::string & );
293 
294  // Fileposition is assumed to have already been set,
295  // get it in string form
296  static std::string filepositionOf( Writable * w );
297 
298  // Execute visitor on each pair of positions in the json value
299  // and the flattened multidimensional array.
300  // Used for writing from the data to JSON and for reading back into
301  // the array from JSON
302  template<
303  typename T,
304  typename Visitor
305  >
306  static void syncMultidimensionalJson(
307  nlohmann::json & j,
308  Offset const & offset,
309  Extent const & extent,
310  Extent const & multiplicator,
311  Visitor visitor,
312  T * data,
313  size_t currentdim = 0
314  );
315 
316  // multiplicators: an array [m_0,...,m_n] s.t.
317  // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n]
318  // (m_n = 1)
319  // essentially: m_i = \prod_{j=0}^{i-1} extent_j
320  static Extent getMultiplicators( Extent const & extent );
321 
322  static nlohmann::json initializeNDArray( Extent const & extent );
323 
324  static Extent getExtent( nlohmann::json & j );
325 
326 
327  // remove single '/' in the beginning and end of a string
328  static std::string removeSlashes( std::string );
329 
330  template< typename KeyT >
331  static bool hasKey(
332  nlohmann::json &,
333  KeyT && key
334  );
335 
336  // make sure that the given path exists in proper form in
337  // the passed json value
338  static void ensurePath(
339  nlohmann::json * json,
340  std::string path
341  );
342 
343  // In order not to insert the same file name into the data structures
344  // with a new pointer (e.g. when reopening), search for a possibly
345  // existing old pointer. Construct a new pointer only upon failure.
346  // The bool is true iff the pointer has been newly-created.
347  // The iterator is an iterator for m_files
348  std::tuple<
349  File,
350  std::unordered_map<
351  Writable *,
352  File
353  >::iterator,
354  bool
355  > getPossiblyExisting(
356  std::string file
357  );
358 
359  // get the json value representing the whole file, possibly reading
360  // from disk
361  std::shared_ptr< nlohmann::json > obtainJsonContents( File );
362 
363  // get the json value at the writable's fileposition
364  nlohmann::json & obtainJsonContents( Writable * writable );
365 
366  // write to disk the json contents associated with the file
367  // remove from m_dirty if unsetDirty == true
368  void putJsonContents(
369  File,
370  bool unsetDirty = true
371  );
372 
373  // figure out the file position of the writable
374  // (preferring the parent's file position) and extend it
375  // by extend. return the modified file position.
376  std::shared_ptr< JSONFilePosition > setAndGetFilePosition(
377  Writable *,
378  std::string extend
379  );
380 
381  // figure out the file position of the writable
382  // (preferring the parent's file position)
383  // only modify the writable's fileposition when specified
384  std::shared_ptr< JSONFilePosition > setAndGetFilePosition(
385  Writable *,
386  bool write = true
387  );
388 
389  // get the writable's containing file
390  // if the parent is associated with another file,
391  // associate the writable with that file and return it
392  File refreshFileFromParent( Writable * writable );
393 
394  void associateWithFile(
395  Writable * writable,
396  File
397  );
398 
399  // need to check the name too in order to exclude "attributes" key
400  static bool isGroup( nlohmann::json::const_iterator it );
401 
402  static bool isDataset( nlohmann::json const & j );
403 
404 
405  // check whether the json reference contains a valid dataset
406  template< typename Param >
407  void verifyDataset(
408  Param const & parameters,
409  nlohmann::json &
410  );
411 
412  static nlohmann::json platformSpecifics( );
413 
414  struct DatasetWriter
415  {
416  template< typename T >
417  void operator()(
418  nlohmann::json & json,
419  const Parameter< Operation::WRITE_DATASET > & parameters
420  );
421 
422  template< int n >
423  void operator()(
424  nlohmann::json & json,
425  const Parameter< Operation::WRITE_DATASET > & parameters
426  );
427 
428  };
429 
430  struct DatasetReader
431  {
432  template< typename T >
433  void operator()(
434  nlohmann::json & json,
436  );
437 
438  template< int n >
439  void operator()(
440  nlohmann::json & json,
442  );
443  };
444 
445  struct AttributeWriter
446  {
447  template< typename T >
448  void operator()(
449  nlohmann::json &,
450  Attribute::resource const &
451  );
452 
453  template< int n >
454  void operator()(
455  nlohmann::json &,
456  Attribute::resource const &
457  );
458 
459  };
460 
461  struct AttributeReader
462  {
463  template< typename T >
464  void operator()(
465  nlohmann::json &,
467  );
468 
469  template< int n >
470  void operator()(
471  nlohmann::json &,
473  );
474 
475  };
476 
477  template< typename T >
478  struct CppToJSON
479  {
480  nlohmann::json operator()( T const & );
481  };
482 
483  template< typename T >
484  struct CppToJSON< std::vector< T>>
485  {
486  nlohmann::json operator()( std::vector< T > const & );
487  };
488 
489  template< typename T, int n >
490  struct CppToJSON<
491  std::array<
492  T,
493  n>>
494  {
495  nlohmann::json operator()(
496  std::array<
497  T,
498  n
499  > const &
500  );
501  };
502 
503  template<
504  typename T,
505  typename Enable = T
506  >
507  struct JsonToCpp
508  {
509  T operator()( nlohmann::json const & );
510  };
511 
512  template< typename T >
513  struct JsonToCpp< std::vector< T > >
514  {
515  std::vector< T > operator()( nlohmann::json const & );
516  };
517 
518  template< typename T, int n >
519  struct JsonToCpp<
520  std::array<
521  T,
522  n
523  >
524  >
525  {
526  std::array<
527  T,
528  n
529  > operator()( nlohmann::json const & );
530  };
531 
532  template< typename T >
533  struct JsonToCpp<
534  T,
535  typename std::enable_if<
536  std::is_floating_point<
537  T
538  >::value
539  >::type
540  >
541  {
542  T operator()( nlohmann::json const & );
543  };
544  };
545 
546 } // openPMD
AccessType
File access mode to use during IO.
Definition: AccessType.hpp:28
Definition: JSONIOHandlerImpl.hpp:67
STL namespace.
Definition: JSONIOHandlerImpl.hpp:57
Definition: JSONIOHandlerImpl.hpp:152
Interface for communicating between logical and physically persistent data.
Definition: AbstractIOHandler.hpp:68
Public definitions of openPMD-api.
Definition: Date.cpp:28
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:55
Definition: AbstractIOHandlerImpl.hpp:34