openPMD-api
Container.hpp
1 /* Copyright 2017-2021 Fabian Koller and Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/Error.hpp"
24 #include "openPMD/backend/Attributable.hpp"
25 
26 #include <initializer_list>
27 #include <map>
28 #include <set>
29 #include <stdexcept>
30 #include <string>
31 #include <type_traits>
32 #include <utility>
33 
34 // expose private and protected members for invasive testing
35 #ifndef OPENPMD_protected
36 #define OPENPMD_protected protected:
37 #endif
38 
39 namespace openPMD
40 {
41 namespace traits
42 {
49  template <typename U>
50  struct GenerationPolicy
51  {
52  template <typename T>
53  void operator()(T &)
54  {}
55  };
56 } // namespace traits
57 
58 namespace internal
59 {
60  class SeriesData;
61  template <typename>
63 
64  template <
65  typename T,
66  typename T_key = std::string,
67  typename T_container = std::map<T_key, T> >
69  {
70  public:
71  using InternalContainer = T_container;
72 
76  InternalContainer m_container;
77 
78  ContainerData() = default;
79 
80  ContainerData(ContainerData const &) = delete;
81  ContainerData(ContainerData &&) = delete;
82 
83  ContainerData &operator=(ContainerData const &) = delete;
84  ContainerData &operator=(ContainerData &&) = delete;
85  };
86 } // namespace internal
87 
88 namespace detail
89 {
90  /*
91  * This converts the key (first parameter) to its string name within the
92  * openPMD hierarchy.
93  * If the key is found to be equal to RecordComponent::SCALAR, the parentKey
94  * will be returned, adding RecordComponent::SCALAR to its back.
95  * Reason: Scalar record components do not link their containing record as
96  * parent, but rather the parent's parent, so the own key within the
97  * "apparent" parent must be given as two steps.
98  */
99  template <typename T>
100  std::vector<std::string>
101  keyAsString(T &&key, std::vector<std::string> const &parentKey)
102  {
103  (void)parentKey;
104  return {std::to_string(std::forward<T>(key))};
105  }
106 
107  // moved to a *.cpp file so we don't need to include RecordComponent.hpp
108  // here
109  template <>
110  std::vector<std::string> keyAsString<std::string const &>(
111  std::string const &key, std::vector<std::string> const &parentKey);
112 
113  template <>
114  std::vector<std::string> keyAsString<std::string>(
115  std::string &&key, std::vector<std::string> const &parentKey);
116 } // namespace detail
117 
127 template <
128  typename T,
129  typename T_key = std::string,
130  typename T_container = std::map<T_key, T> >
131 class Container : public Attributable
132 {
133  static_assert(
134  std::is_base_of<Attributable, T>::value,
135  "Type of container element must be derived from Writable");
136 
137  friend class Iteration;
138  friend class ParticleSpecies;
139  friend class ParticlePatches;
140  friend class internal::SeriesData;
141  friend class Series;
142  template <typename>
143  friend class internal::EraseStaleEntries;
144  friend class SeriesIterator;
145 
146 protected:
148  using InternalContainer = T_container;
149 
150  std::shared_ptr<ContainerData> m_containerData{new ContainerData()};
151 
152  inline void setData(std::shared_ptr<ContainerData> containerData)
153  {
154  m_containerData = std::move(containerData);
155  Attributable::setData(m_containerData);
156  }
157 
158  inline InternalContainer const &container() const
159  {
160  return m_containerData->m_container;
161  }
162 
163  inline InternalContainer &container()
164  {
165  return m_containerData->m_container;
166  }
167 
168 public:
169  using key_type = typename InternalContainer::key_type;
170  using mapped_type = typename InternalContainer::mapped_type;
171  using value_type = typename InternalContainer::value_type;
172  using size_type = typename InternalContainer::size_type;
173  using difference_type = typename InternalContainer::difference_type;
174  using allocator_type = typename InternalContainer::allocator_type;
175  using reference = typename InternalContainer::reference;
176  using const_reference = typename InternalContainer::const_reference;
177  using pointer = typename InternalContainer::pointer;
178  using const_pointer = typename InternalContainer::const_pointer;
179  using iterator = typename InternalContainer::iterator;
180  using const_iterator = typename InternalContainer::const_iterator;
181 
182  iterator begin() noexcept
183  {
184  return container().begin();
185  }
186  const_iterator begin() const noexcept
187  {
188  return container().begin();
189  }
190  const_iterator cbegin() const noexcept
191  {
192  return container().cbegin();
193  }
194 
195  iterator end() noexcept
196  {
197  return container().end();
198  }
199  const_iterator end() const noexcept
200  {
201  return container().end();
202  }
203  const_iterator cend() const noexcept
204  {
205  return container().cend();
206  }
207 
208  bool empty() const noexcept
209  {
210  return container().empty();
211  }
212 
213  size_type size() const noexcept
214  {
215  return container().size();
216  }
217 
224  void clear()
225  {
226  if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
227  throw std::runtime_error(
228  "Can not clear a container in a read-only Series.");
229 
230  clear_unchecked();
231  }
232 
233  std::pair<iterator, bool> insert(value_type const &value)
234  {
235  return container().insert(value);
236  }
237  template <class P>
238  std::pair<iterator, bool> insert(P &&value)
239  {
240  return container().insert(value);
241  }
242  iterator insert(const_iterator hint, value_type const &value)
243  {
244  return container().insert(hint, value);
245  }
246  template <class P>
247  iterator insert(const_iterator hint, P &&value)
248  {
249  return container().insert(hint, value);
250  }
251  template <class InputIt>
252  void insert(InputIt first, InputIt last)
253  {
254  container().insert(first, last);
255  }
256  void insert(std::initializer_list<value_type> ilist)
257  {
258  container().insert(ilist);
259  }
260 
261  void swap(Container &other)
262  {
263  container().swap(other.m_container);
264  }
265 
266  mapped_type &at(key_type const &key)
267  {
268  return container().at(key);
269  }
270  mapped_type const &at(key_type const &key) const
271  {
272  return container().at(key);
273  }
274 
285  virtual mapped_type &operator[](key_type const &key)
286  {
287  auto it = container().find(key);
288  if (it != container().end())
289  return it->second;
290  else
291  {
292  if (IOHandler()->m_seriesStatus !=
294  Access::READ_ONLY == IOHandler()->m_frontendAccess)
295  {
296  auxiliary::OutOfRangeMsg const out_of_range_msg;
297  throw std::out_of_range(out_of_range_msg(key));
298  }
299 
300  T t = T();
301  t.linkHierarchy(writable());
302  auto &ret = container().insert({key, std::move(t)}).first->second;
303  ret.writable().ownKeyWithinParent =
304  detail::keyAsString(key, writable().ownKeyWithinParent);
306  gen(ret);
307  return ret;
308  }
309  }
320  virtual mapped_type &operator[](key_type &&key)
321  {
322  auto it = container().find(key);
323  if (it != container().end())
324  return it->second;
325  else
326  {
327  if (IOHandler()->m_seriesStatus !=
329  Access::READ_ONLY == IOHandler()->m_frontendAccess)
330  {
331  auxiliary::OutOfRangeMsg out_of_range_msg;
332  throw std::out_of_range(out_of_range_msg(key));
333  }
334 
335  T t = T();
336  t.linkHierarchy(writable());
337  auto &ret = container().insert({key, std::move(t)}).first->second;
338  ret.writable().ownKeyWithinParent = detail::keyAsString(
339  std::move(key), writable().ownKeyWithinParent);
341  gen(ret);
342  return ret;
343  }
344  }
345 
346  iterator find(key_type const &key)
347  {
348  return container().find(key);
349  }
350  const_iterator find(key_type const &key) const
351  {
352  return container().find(key);
353  }
354 
360  size_type count(key_type const &key) const
361  {
362  return container().count(key);
363  }
364 
371  bool contains(key_type const &key) const
372  {
373  return container().find(key) != container().end();
374  }
375 
384  virtual size_type erase(key_type const &key)
385  {
386  if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
387  throw std::runtime_error(
388  "Can not erase from a container in a read-only Series.");
389 
390  auto res = container().find(key);
391  if (res != container().end() && res->second.written())
392  {
394  pDelete.path = ".";
395  IOHandler()->enqueue(IOTask(&res->second, pDelete));
396  IOHandler()->flush(internal::defaultFlushParams);
397  }
398  return container().erase(key);
399  }
400 
402  virtual iterator erase(iterator res)
403  {
404  if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
405  throw std::runtime_error(
406  "Can not erase from a container in a read-only Series.");
407 
408  if (res != container().end() && res->second.written())
409  {
411  pDelete.path = ".";
412  IOHandler()->enqueue(IOTask(&res->second, pDelete));
413  IOHandler()->flush(internal::defaultFlushParams);
414  }
415  return container().erase(res);
416  }
418  // virtual iterator erase(const_iterator first, const_iterator last)
419 
420  template <class... Args>
421  auto emplace(Args &&...args)
422  -> decltype(InternalContainer().emplace(std::forward<Args>(args)...))
423  {
424  return container().emplace(std::forward<Args>(args)...);
425  }
426 
427  // clang-format off
428 OPENPMD_protected
429  // clang-format on
430 
431  Container(std::shared_ptr<ContainerData> containerData)
432  : Attributable{containerData}, m_containerData{std::move(containerData)}
433  {}
434 
435  void clear_unchecked()
436  {
437  if (written())
438  throw std::runtime_error(
439  "Clearing a written container not (yet) implemented.");
440 
441  container().clear();
442  }
443 
444  virtual void
445  flush(std::string const &path, internal::FlushParams const &flushParams)
446  {
447  if (!written())
448  {
450  pCreate.path = path;
451  IOHandler()->enqueue(IOTask(this, pCreate));
452  }
453 
454  flushAttributes(flushParams);
455  }
456 
457  // clang-format off
458 OPENPMD_private
459  // clang-format on
460 
461  Container() : Attributable{nullptr}
462  {
463  Attributable::setData(m_containerData);
464  }
465 };
466 
467 namespace internal
468 {
478  template <typename Container_t>
479  class EraseStaleEntries
480  {
481  using BareContainer_t =
482  typename std::remove_reference<Container_t>::type;
483  using key_type = typename BareContainer_t::key_type;
484  using mapped_type = typename BareContainer_t::mapped_type;
485  std::set<key_type> m_accessedKeys;
486  /*
487  * Note: Putting a copy here leads to weird bugs due to destructors
488  * being called too eagerly upon destruction.
489  * Should be avoidable by extending the frontend redesign to the
490  * Container class template
491  * (https://github.com/openPMD/openPMD-api/pull/886)
492  */
493  Container_t m_originalContainer;
494 
495  public:
496  explicit EraseStaleEntries(Container_t &container_in)
497  : m_originalContainer(container_in)
498  {}
499 
500  explicit EraseStaleEntries(BareContainer_t &&container_in)
501  : m_originalContainer(std::move(container_in))
502  {}
503 
504  EraseStaleEntries(EraseStaleEntries &&) = default;
505  EraseStaleEntries &operator=(EraseStaleEntries &&) = default;
506 
507  template <typename K>
508  mapped_type &operator[](K &&k)
509  {
510  m_accessedKeys.insert(k); // copy
511  return m_originalContainer[std::forward<K>(k)];
512  }
513 
514  template <typename K>
515  mapped_type &at(K &&k)
516  {
517  m_accessedKeys.insert(k); // copy
518  return m_originalContainer.at(std::forward<K>(k));
519  }
520 
526  template <typename K>
527  void forget(K &&k)
528  {
529  m_accessedKeys.erase(std::forward<K>(k));
530  }
531 
533  {
534  auto &map = m_originalContainer.container();
535  using iterator_t = typename BareContainer_t::const_iterator;
536  std::vector<iterator_t> deleteMe;
537  deleteMe.reserve(map.size() - m_accessedKeys.size());
538  for (iterator_t it = map.begin(); it != map.end(); ++it)
539  {
540  auto lookup = m_accessedKeys.find(it->first);
541  if (lookup == m_accessedKeys.end())
542  {
543  deleteMe.push_back(it);
544  }
545  }
546  for (auto &it : deleteMe)
547  {
548  map.erase(it);
549  }
550  }
551  };
552 } // namespace internal
553 } // namespace openPMD
Return an error string for read-only access.
Definition: OutOfRangeMsg.hpp:36
Definition: ParticlePatches.hpp:31
Self-contained description of a single IO operation.
Definition: IOTask.hpp:695
virtual mapped_type & operator[](key_type const &key)
Access the value that is mapped to a key equivalent to key, creating it if such key does not exist al...
Definition: Container.hpp:285
Container Element Creation Policy.
Definition: Attributable.hpp:47
Logical compilation of data from one snapshot (e.g.
Definition: Iteration.hpp:126
auto emplace(Args &&...args) -> decltype(InternalContainer().emplace(std::forward< Args >(args)...))
Definition: Container.hpp:421
bool contains(key_type const &key) const
Checks if there is an element with a key equivalent to an exiting key in the container.
Definition: Container.hpp:371
virtual size_type erase(key_type const &key)
Remove a single element from the container and (if written) from disk.
Definition: Container.hpp:384
Implementation for the root level of the openPMD hierarchy.
Definition: Series.hpp:186
Definition: Container.hpp:68
Definition: Container.cpp:51
Public definitions of openPMD-api.
void clear()
Remove all objects from the container and (if written) from disk.
Definition: Container.hpp:224
void forget(K &&k)
Remove key from the list of accessed keys.
Definition: Container.hpp:527
Data members for Series.
Definition: Series.hpp:67
Definition: Attributable.hpp:56
Definition: ReadIterations.hpp:54
All objects in the openPMD object model are temporarily mutable to allow inserting newly-parsed data...
InternalContainer m_container
The wrapped container holding all the actual data, e.g.
Definition: Container.hpp:76
Parameters recursively passed through the openPMD hierarchy when flushing.
Definition: AbstractIOHandler.hpp:84
size_type count(key_type const &key) const
This returns either 1 if the key is found in the container of 0 if not.
Definition: Container.hpp:360
virtual iterator erase(iterator res)
Definition: Container.hpp:402
Definition: ParticleSpecies.hpp:33
This class wraps a Container and forwards operator[]() and at() to it.
Definition: Container.hpp:62
Map-like container that enforces openPMD requirements and handles IO.
Definition: Container.hpp:131
Open Series as read-only, fails if Series is not found.
virtual mapped_type & operator[](key_type &&key)
Access the value that is mapped to a key equivalent to key, creating it if such key does not exist al...
Definition: Container.hpp:320
Layer to manage storage of attributes associated with file objects.
Definition: Attributable.hpp:93