openPMD-api
Series.hpp
1 /* Copyright 2017-2021 Fabian Koller, Axel Huebl
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/IO/AbstractIOHandler.hpp"
24 #include "openPMD/IO/Access.hpp"
25 #include "openPMD/IO/Format.hpp"
26 #include "openPMD/Iteration.hpp"
27 #include "openPMD/IterationEncoding.hpp"
28 #include "openPMD/Streaming.hpp"
29 #include "openPMD/WriteIterations.hpp"
30 #include "openPMD/auxiliary/Variant.hpp"
31 #include "openPMD/backend/Attributable.hpp"
32 #include "openPMD/backend/Container.hpp"
33 #include "openPMD/backend/ParsePreference.hpp"
34 #include "openPMD/config.hpp"
35 #include "openPMD/version.hpp"
36 
37 #if openPMD_HAVE_MPI
38 #include <mpi.h>
39 #endif
40 
41 #include <cstdint> // uint64_t
42 #include <deque>
43 #include <map>
44 #include <optional>
45 #include <set>
46 #include <string>
47 
48 // expose private and protected members for invasive testing
49 #ifndef OPENPMD_private
50 #define OPENPMD_private private:
51 #endif
52 
53 namespace openPMD
54 {
55 class ReadIterations;
56 class Series;
57 class Series;
58 
59 namespace internal
60 {
68  {
69  public:
70  explicit SeriesData() = default;
71 
72  virtual ~SeriesData();
73 
74  SeriesData(SeriesData const &) = delete;
75  SeriesData(SeriesData &&) = delete;
76 
77  SeriesData &operator=(SeriesData const &) = delete;
78  SeriesData &operator=(SeriesData &&) = delete;
79 
80  using IterationIndex_t = Iteration::IterationIndex_t;
82  IterationsContainer_t iterations{};
83 
90  std::optional<WriteIterations> m_writeIterations;
96  std::set<IterationIndex_t> m_currentlyActiveIterations;
106  std::optional<std::string> m_overrideFilebasedFilename;
113  std::string m_name;
118  std::string m_filenamePrefix;
122  std::string m_filenamePostfix;
127  std::string m_filenameExtension;
149  StepStatus m_stepStatus = StepStatus::NoStep;
153  bool m_parseLazily = false;
161  bool m_lastFlushSuccessful = false;
162 
169  std::optional<ParsePreference> m_parsePreference;
170 
171  void close();
172  }; // SeriesData
173 
174  class SeriesInternal;
175 } // namespace internal
176 
186 class Series : public Attributable
187 {
188  friend class Attributable;
189  friend class Iteration;
190  friend class Writable;
191  friend class SeriesIterator;
192  friend class internal::SeriesData;
193  friend class WriteIterations;
194 
195 protected:
196  // Should not be called publicly, only by implementing classes
197  Series(std::shared_ptr<internal::SeriesData>);
198 
199 public:
200  explicit Series();
201 
202 #if openPMD_HAVE_MPI
203  Series(
204  std::string const &filepath,
205  Access at,
206  MPI_Comm comm,
207  std::string const &options = "{}");
208 #endif
209 
219  Series(
220  std::string const &filepath,
221  Access at,
222  std::string const &options = "{}");
223 
224  virtual ~Series() = default;
225 
229  using IterationIndex_t = Iteration::IterationIndex_t;
234  IterationsContainer_t iterations;
235 
242  operator bool() const;
243 
249  std::string openPMD() const;
258  Series &setOpenPMD(std::string const &openPMD);
259 
265  uint32_t openPMDextension() const;
274  Series &setOpenPMDextension(uint32_t openPMDextension);
275 
280  std::string basePath() const;
288  Series &setBasePath(std::string const &basePath);
289 
295  std::string meshesPath() const;
305  Series &setMeshesPath(std::string const &meshesPath);
306 
312  std::string particlesPath() const;
322  Series &setParticlesPath(std::string const &particlesPath);
323 
329  std::string author() const;
336  Series &setAuthor(std::string const &author);
337 
343  std::string software() const;
352  Series &setSoftware(
353  std::string const &newName,
354  std::string const &newVersion = std::string("unspecified"));
355 
361  std::string softwareVersion() const;
371  [[deprecated(
372  "Set the version with the second argument of setSoftware()")]] Series &
373  setSoftwareVersion(std::string const &softwareVersion);
374 
379  std::string date() const;
385  Series &setDate(std::string const &date);
386 
392  std::string softwareDependencies() const;
400  Series &setSoftwareDependencies(std::string const &newSoftwareDependencies);
401 
407  std::string machine() const;
414  Series &setMachine(std::string const &newMachine);
415 
419  IterationEncoding iterationEncoding() const;
437  Series &setIterationEncoding(IterationEncoding iterationEncoding);
438 
444  std::string iterationFormat() const;
456  Series &setIterationFormat(std::string const &iterationFormat);
457 
461  std::string name() const;
462 
469  Series &setName(std::string const &name);
470 
477  std::string backend() const;
478 
486  void flush(std::string backendConfig = "{}");
487 
498  ReadIterations readIterations();
499 
512  WriteIterations writeIterations();
513 
524  void close();
525 
526  // clang-format off
527 OPENPMD_private
528  // clang-format on
529 
530  static constexpr char const *const BASEPATH = "/data/%T/";
531 
532  struct ParsedInput;
533  using iterations_t = decltype(internal::SeriesData::iterations);
534  using iterations_iterator = iterations_t::iterator;
535 
536  std::shared_ptr<internal::SeriesData> m_series = nullptr;
537 
538  inline internal::SeriesData &get()
539  {
540  if (m_series)
541  {
542  return *m_series;
543  }
544  else
545  {
546  throw std::runtime_error(
547  "[Series] Cannot use default-constructed Series.");
548  }
549  }
550 
551  inline internal::SeriesData const &get() const
552  {
553  if (m_series)
554  {
555  return *m_series;
556  }
557  else
558  {
559  throw std::runtime_error(
560  "[Series] Cannot use default-constructed Series.");
561  }
562  }
563 
564  std::unique_ptr<ParsedInput> parseInput(std::string);
574  template <typename TracingJSON>
575  void parseJsonOptions(TracingJSON &options, ParsedInput &);
576  bool hasExpansionPattern(std::string filenameWithExtension);
577  bool reparseExpansionPattern(std::string filenameWithExtension);
578  void init(std::unique_ptr<AbstractIOHandler>, std::unique_ptr<ParsedInput>);
579  void initDefaults(IterationEncoding, bool initAll = false);
591  std::future<void> flush_impl(
592  iterations_iterator begin,
593  iterations_iterator end,
594  internal::FlushParams flushParams,
595  bool flushIOHandler = true);
596  void flushFileBased(
597  iterations_iterator begin,
598  iterations_iterator end,
599  internal::FlushParams flushParams,
600  bool flushIOHandler = true);
601  /*
602  * Group-based and variable-based iteration layouts share a lot of logic
603  * (realistically, the variable-based iteration layout only throws out
604  * one layer in the hierarchy).
605  * As a convention, methods that deal with both layouts are called
606  * .*GorVBased, short for .*GroupOrVariableBased
607  */
608  void flushGorVBased(
609  iterations_iterator begin,
610  iterations_iterator end,
611  internal::FlushParams flushParams,
612  bool flushIOHandler = true);
613  void flushMeshesPath();
614  void flushParticlesPath();
615  void readFileBased();
616  void readOneIterationFileBased(std::string const &filePath);
630  std::optional<std::deque<IterationIndex_t> > readGorVBased(
631  bool do_always_throw_errors,
632  bool init,
633  std::set<IterationIndex_t> const &ignoreIterations = {});
634  void readBase();
635  std::string iterationFilename(IterationIndex_t i);
636 
637  enum class IterationOpened : bool
638  {
639  HasBeenOpened,
640  RemainsClosed
641  };
642  /*
643  * For use by flushFileBased, flushGorVBased
644  * Open an iteration, but only if necessary.
645  * Only open if the iteration is dirty and if it is not in deferred
646  * parse state.
647  */
648  IterationOpened
649  openIterationIfDirty(IterationIndex_t index, Iteration iteration);
650  /*
651  * Open an iteration. Ensures that the iteration's m_closed status
652  * is set properly and that any files pertaining to the iteration
653  * is opened.
654  * Does not create files when called in CREATE mode.
655  */
656  void openIteration(IterationIndex_t index, Iteration iteration);
657 
662  iterations_iterator indexOf(Iteration const &);
663 
679  AdvanceStatus advance(
680  AdvanceMode mode,
682  iterations_iterator it,
683  Iteration &iteration);
684 
685  AdvanceStatus advance(AdvanceMode mode);
686 
693  void flushStep(bool doFlush);
694 
695  /*
696  * Returns the current content of the /data/snapshot attribute.
697  * (We could also add this to the public API some time)
698  */
699  std::optional<std::vector<IterationIndex_t> > currentSnapshot() const;
700 }; // Series
701 } // namespace openPMD
702 
703 // Make sure that this one is always included if Series.hpp is included,
704 // otherwise Series::readIterations() cannot be used
705 #include "openPMD/ReadIterations.hpp"
bool m_lastFlushSuccessful
This is to avoid that the destructor tries flushing again if an error happened.
Definition: Series.hpp:161
std::string m_filenameExtension
Filename extension as specified by the user.
Definition: Series.hpp:127
std::optional< std::string > m_overrideFilebasedFilename
Needed if reading a single iteration of a file-based series.
Definition: Series.hpp:106
bool m_parseLazily
True if a user opts into lazy parsing.
Definition: Series.hpp:153
std::optional< ParsePreference > m_parsePreference
Remember the preference that the backend specified for parsing.
Definition: Series.hpp:169
Access
File access mode to use during IO.
Definition: Access.hpp:29
Format
File format to use during IO.
Definition: Format.hpp:29
Logical compilation of data from one snapshot (e.g.
Definition: Iteration.hpp:126
StepStatus m_stepStatus
Whether a step is currently active for this iteration.
Definition: Series.hpp:149
std::optional< WriteIterations > m_writeIterations
For each instance of Series, there is only one instance of WriteIterations, stored in this Option...
Definition: Series.hpp:90
Definition: Series.cpp:107
StepStatus
Used in step-based mode (i.e.
Definition: Streaming.hpp:45
IterationEncoding m_iterationEncoding
The iteration encoding used in this series.
Definition: Series.hpp:137
std::string m_filenamePostfix
Filename after the expansion pattern without filename extension.
Definition: Series.hpp:122
Format m_format
Detected IO format (backend).
Definition: Series.hpp:141
std::string m_name
Name of the iteration without filename suffix.
Definition: Series.hpp:113
std::set< IterationIndex_t > m_currentlyActiveIterations
For writing: Remember which iterations have been written in the currently active output step...
Definition: Series.hpp:96
AdvanceStatus
In step-based mode (i.e.
Definition: Streaming.hpp:20
AdvanceMode
In step-based mode (i.e.
Definition: Streaming.hpp:34
Definition: WriteIterations.hpp:53
Reading side of the streaming API.
Definition: ReadIterations.hpp:165
Implementation for the root level of the openPMD hierarchy.
Definition: Series.hpp:186
std::string m_filenamePrefix
Filename leading up to the expansion pattern.
Definition: Series.hpp:118
Public definitions of openPMD-api.
Layer to mirror structure of logical data and persistent data in file.
Definition: Writable.hpp:64
Iteration::IterationIndex_t IterationIndex_t
An unsigned integer type, used to identify Iterations in a Series.
Definition: Series.hpp:229
Data members for Series.
Definition: Series.hpp:67
Definition: Attributable.hpp:56
Definition: ReadIterations.hpp:54
Parameters recursively passed through the openPMD hierarchy when flushing.
Definition: AbstractIOHandler.hpp:84
IterationEncoding
Encoding scheme of an Iterations Series&#39;.
Definition: IterationEncoding.hpp:32
int m_filenamePadding
The padding in file-based iteration encoding.
Definition: Series.hpp:133
Layer to manage storage of attributes associated with file objects.
Definition: Attributable.hpp:93