openPMD-api
Container.hpp
1 /* Copyright 2017-2021 Fabian Koller and Franz Poeschel
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/Error.hpp"
24 #include "openPMD/IO/Access.hpp"
25 #include "openPMD/backend/Attributable.hpp"
26 
27 #include <initializer_list>
28 #include <map>
29 #include <set>
30 #include <stdexcept>
31 #include <string>
32 #include <type_traits>
33 #include <utility>
34 
35 // expose private and protected members for invasive testing
36 #ifndef OPENPMD_protected
37 #define OPENPMD_protected protected:
38 #endif
39 
40 namespace openPMD
41 {
42 namespace traits
43 {
50  template <typename U>
52  {
53  constexpr static bool is_noop = true;
54  template <typename T>
55  void operator()(T &)
56  {}
57  };
58 } // namespace traits
59 
60 namespace internal
61 {
62  class SeriesData;
63  template <typename>
64  class EraseStaleEntries;
65 
66  template <
67  typename T,
68  typename T_key = std::string,
69  typename T_container = std::map<T_key, T> >
70  class ContainerData : virtual public AttributableData
71  {
72  public:
73  using InternalContainer = T_container;
74 
78  InternalContainer m_container;
79 
80  ContainerData() = default;
81 
82  ContainerData(ContainerData const &) = delete;
83  ContainerData(ContainerData &&) = delete;
84 
85  ContainerData &operator=(ContainerData const &) = delete;
86  ContainerData &operator=(ContainerData &&) = delete;
87  };
88 } // namespace internal
89 
99 template <
100  typename T,
101  typename T_key = std::string,
102  typename T_container = std::map<T_key, T> >
103 class Container : virtual public Attributable
104 {
105  static_assert(
106  std::is_base_of<Attributable, T>::value,
107  "Type of container element must be derived from Writable");
108 
109  friend class Iteration;
110  friend class ParticleSpecies;
111  friend class ParticlePatches;
112  friend class internal::SeriesData;
113  friend class Series;
114  template <typename>
115  friend class internal::EraseStaleEntries;
116  friend class SeriesIterator;
117 
118 protected:
120  using InternalContainer = T_container;
121 
122  std::shared_ptr<ContainerData> m_containerData;
123 
124  inline void setData(std::shared_ptr<ContainerData> containerData)
125  {
126  m_containerData = std::move(containerData);
127  Attributable::setData(m_containerData);
128  }
129 
130  inline InternalContainer const &container() const
131  {
132  return m_containerData->m_container;
133  }
134 
135  inline InternalContainer &container()
136  {
137  return m_containerData->m_container;
138  }
139 
140 public:
141  using key_type = typename InternalContainer::key_type;
142  using mapped_type = typename InternalContainer::mapped_type;
143  using value_type = typename InternalContainer::value_type;
144  using size_type = typename InternalContainer::size_type;
145  using difference_type = typename InternalContainer::difference_type;
146  using allocator_type = typename InternalContainer::allocator_type;
147  using reference = typename InternalContainer::reference;
148  using const_reference = typename InternalContainer::const_reference;
149  using pointer = typename InternalContainer::pointer;
150  using const_pointer = typename InternalContainer::const_pointer;
151  using iterator = typename InternalContainer::iterator;
152  using const_iterator = typename InternalContainer::const_iterator;
153  using reverse_iterator = typename InternalContainer::reverse_iterator;
154  using const_reverse_iterator =
155  typename InternalContainer::const_reverse_iterator;
156 
157  iterator begin() noexcept
158  {
159  return container().begin();
160  }
161  const_iterator begin() const noexcept
162  {
163  return container().begin();
164  }
165  const_iterator cbegin() const noexcept
166  {
167  return container().cbegin();
168  }
169 
170  iterator end() noexcept
171  {
172  return container().end();
173  }
174  const_iterator end() const noexcept
175  {
176  return container().end();
177  }
178  const_iterator cend() const noexcept
179  {
180  return container().cend();
181  }
182 
183  reverse_iterator rbegin() noexcept
184  {
185  return container().rbegin();
186  }
187  const_reverse_iterator rbegin() const noexcept
188  {
189  return container().rbegin();
190  }
191  const_reverse_iterator crbegin() const noexcept
192  {
193  return container().crbegin();
194  }
195 
196  reverse_iterator rend() noexcept
197  {
198  return container().rend();
199  }
200  const_reverse_iterator rend() const noexcept
201  {
202  return container().rend();
203  }
204  const_reverse_iterator crend() const noexcept
205  {
206  return container().crend();
207  }
208 
209  bool empty() const noexcept
210  {
211  return container().empty();
212  }
213 
214  size_type size() const noexcept
215  {
216  return container().size();
217  }
218 
225  void clear()
226  {
227  if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
228  throw std::runtime_error(
229  "Can not clear a container in a read-only Series.");
230 
231  clear_unchecked();
232  }
233 
234  std::pair<iterator, bool> insert(value_type const &value)
235  {
236  return container().insert(value);
237  }
238  std::pair<iterator, bool> insert(value_type &&value)
239  {
240  return container().insert(value);
241  }
242  iterator insert(const_iterator hint, value_type const &value)
243  {
244  return container().insert(hint, value);
245  }
246  iterator insert(const_iterator hint, value_type &&value)
247  {
248  return container().insert(hint, value);
249  }
250  template <class InputIt>
251  void insert(InputIt first, InputIt last)
252  {
253  container().insert(first, last);
254  }
255  void insert(std::initializer_list<value_type> ilist)
256  {
257  container().insert(ilist);
258  }
259 
260  void swap(Container &other)
261  {
262  container().swap(other.m_container);
263  }
264 
265  mapped_type &at(key_type const &key)
266  {
267  return container().at(key);
268  }
269  mapped_type const &at(key_type const &key) const
270  {
271  return container().at(key);
272  }
273 
284  mapped_type &operator[](key_type const &key)
285  {
286  auto it = container().find(key);
287  if (it != container().end())
288  return it->second;
289  else
290  {
291  if (IOHandler()->m_seriesStatus !=
293  access::readOnly(IOHandler()->m_frontendAccess))
294  {
295  auxiliary::OutOfRangeMsg const out_of_range_msg;
296  throw std::out_of_range(out_of_range_msg(key));
297  }
298 
299  T t = T();
300  t.linkHierarchy(writable());
301  auto &ret = container().insert({key, std::move(t)}).first->second;
302  if constexpr (std::is_same_v<T_key, std::string>)
303  {
304  ret.writable().ownKeyWithinParent = key;
305  }
306  else
307  {
308  ret.writable().ownKeyWithinParent = std::to_string(key);
309  }
311  gen(ret);
312  return ret;
313  }
314  }
325  mapped_type &operator[](key_type &&key)
326  {
327  auto it = container().find(key);
328  if (it != container().end())
329  return it->second;
330  else
331  {
332  if (IOHandler()->m_seriesStatus !=
334  access::readOnly(IOHandler()->m_frontendAccess))
335  {
336  auxiliary::OutOfRangeMsg out_of_range_msg;
337  throw std::out_of_range(out_of_range_msg(key));
338  }
339 
340  T t = T();
341  t.linkHierarchy(writable());
342  auto &ret = container().insert({key, std::move(t)}).first->second;
343  if constexpr (std::is_same_v<T_key, std::string>)
344  {
345  ret.writable().ownKeyWithinParent = std::move(key);
346  }
347  else
348  {
349  ret.writable().ownKeyWithinParent =
350  std::to_string(std::move(key));
351  }
353  gen(ret);
354  return ret;
355  }
356  }
357 
358  iterator find(key_type const &key)
359  {
360  return container().find(key);
361  }
362  const_iterator find(key_type const &key) const
363  {
364  return container().find(key);
365  }
366 
372  size_type count(key_type const &key) const
373  {
374  return container().count(key);
375  }
376 
383  bool contains(key_type const &key) const
384  {
385  return container().find(key) != container().end();
386  }
387 
396  size_type erase(key_type const &key)
397  {
398  if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
399  throw std::runtime_error(
400  "Can not erase from a container in a read-only Series.");
401 
402  auto res = container().find(key);
403  if (res != container().end() && res->second.written())
404  {
406  pDelete.path = ".";
407  IOHandler()->enqueue(IOTask(&res->second, pDelete));
408  IOHandler()->flush(internal::defaultFlushParams);
409  }
410  return container().erase(key);
411  }
412 
414  iterator erase(iterator res)
415  {
416  if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
417  throw std::runtime_error(
418  "Can not erase from a container in a read-only Series.");
419 
420  if (res != container().end() && res->second.written())
421  {
423  pDelete.path = ".";
424  IOHandler()->enqueue(IOTask(&res->second, pDelete));
425  IOHandler()->flush(internal::defaultFlushParams);
426  }
427  return container().erase(res);
428  }
430  // virtual iterator erase(const_iterator first, const_iterator last)
431 
432  template <class... Args>
433  auto emplace(Args &&...args)
434  -> decltype(InternalContainer().emplace(std::forward<Args>(args)...))
435  {
436  return container().emplace(std::forward<Args>(args)...);
437  }
438 
439  // clang-format off
440 OPENPMD_protected
441  // clang-format on
442 
443  void clear_unchecked()
444  {
445  if (written())
446  throw std::runtime_error(
447  "Clearing a written container not (yet) implemented.");
448 
449  container().clear();
450  }
451 
452  virtual void
453  flush(std::string const &path, internal::FlushParams const &flushParams)
454  {
455  if (!written())
456  {
457  Parameter<Operation::CREATE_PATH> pCreate;
458  pCreate.path = path;
459  IOHandler()->enqueue(IOTask(this, pCreate));
460  }
461 
462  flushAttributes(flushParams);
463  }
464 
465  Container() : Attributable(NoInit())
466  {
467  setData(std::make_shared<ContainerData>());
468  }
469 
470  Container(NoInit) : Attributable(NoInit())
471  {}
472 
473 public:
474  /*
475  * Need to define these manually due to the virtual inheritance from
476  * Attributable.
477  * Otherwise, they would only run from the most derived class
478  * if explicitly called.
479  * If not defining these, a user could destroy copy/move constructors/
480  * assignment operators by deriving from any class that has a virtual
481  * Attributable somewhere.
482  * Care must be taken in move constructors/assignment operators to not move
483  * multiple times (which could happen in diamond inheritance situations).
484  */
485 
486  Container(Container const &other) : Attributable(NoInit())
487  {
488  m_attri = other.m_attri;
489  m_containerData = other.m_containerData;
490  }
491 
492  Container(Container &&other) : Attributable(NoInit())
493  {
494  if (other.m_attri)
495  {
496  m_attri = std::move(other.m_attri);
497  }
498  m_containerData = std::move(other.m_containerData);
499  }
500 
501  Container &operator=(Container const &other)
502  {
503  m_attri = other.m_attri;
504  m_containerData = other.m_containerData;
505  return *this;
506  }
507 
508  Container &operator=(Container &&other)
509  {
510  if (other.m_attri)
511  {
512  m_attri = std::move(other.m_attri);
513  }
514  m_containerData = std::move(other.m_containerData);
515  return *this;
516  }
517 };
518 
519 namespace internal
520 {
530  template <typename Container_t>
532  {
533  using BareContainer_t =
534  typename std::remove_reference<Container_t>::type;
535  using key_type = typename BareContainer_t::key_type;
536  using mapped_type = typename BareContainer_t::mapped_type;
537  std::set<key_type> m_accessedKeys;
538  /*
539  * Note: Putting a copy here leads to weird bugs due to destructors
540  * being called too eagerly upon destruction.
541  * Should be avoidable by extending the frontend redesign to the
542  * Container class template
543  * (https://github.com/openPMD/openPMD-api/pull/886)
544  */
545  Container_t m_originalContainer;
546 
547  public:
548  explicit EraseStaleEntries(Container_t &container_in)
549  : m_originalContainer(container_in)
550  {}
551 
552  explicit EraseStaleEntries(BareContainer_t &&container_in)
553  : m_originalContainer(std::move(container_in))
554  {}
555 
556  EraseStaleEntries(EraseStaleEntries &&) = default;
557  EraseStaleEntries &operator=(EraseStaleEntries &&) = default;
558 
559  template <typename K>
560  mapped_type &operator[](K &&k)
561  {
562  m_accessedKeys.insert(k); // copy
563  return m_originalContainer[std::forward<K>(k)];
564  }
565 
566  template <typename K>
567  mapped_type &at(K &&k)
568  {
569  m_accessedKeys.insert(k); // copy
570  return m_originalContainer.at(std::forward<K>(k));
571  }
572 
578  template <typename K>
579  void forget(K &&k)
580  {
581  m_accessedKeys.erase(std::forward<K>(k));
582  }
583 
585  {
586  auto &map = m_originalContainer.container();
587  using iterator_t =
588  typename BareContainer_t::InternalContainer::const_iterator;
589  std::vector<iterator_t> deleteMe;
590  deleteMe.reserve(map.size() - m_accessedKeys.size());
591  for (iterator_t it = map.begin(); it != map.end(); ++it)
592  {
593  auto lookup = m_accessedKeys.find(it->first);
594  if (lookup == m_accessedKeys.end())
595  {
596  deleteMe.push_back(it);
597  }
598  }
599  for (auto &it : deleteMe)
600  {
601  map.erase(it);
602  }
603  }
604  };
605 } // namespace internal
606 } // namespace openPMD
std::future< void > flush(internal::FlushParams const &)
Process operations in queue according to FIFO.
Definition: AbstractIOHandler.cpp:28
virtual void enqueue(IOTask const &iotask)
Add provided task to queue according to FIFO.
Definition: AbstractIOHandler.hpp:232
Layer to manage storage of attributes associated with file objects.
Definition: Attributable.hpp:101
Map-like container that enforces openPMD requirements and handles IO.
Definition: Container.hpp:104
mapped_type & operator[](key_type const &key)
Access the value that is mapped to a key equivalent to key, creating it if such key does not exist al...
Definition: Container.hpp:284
void clear()
Remove all objects from the container and (if written) from disk.
Definition: Container.hpp:225
size_type erase(key_type const &key)
Remove a single element from the container and (if written) from disk.
Definition: Container.hpp:396
size_type count(key_type const &key) const
This returns either 1 if the key is found in the container of 0 if not.
Definition: Container.hpp:372
bool contains(key_type const &key) const
Checks if there is an element with a key equivalent to an exiting key in the container.
Definition: Container.hpp:383
iterator erase(iterator res)
Definition: Container.hpp:414
auto emplace(Args &&...args) -> decltype(InternalContainer().emplace(std::forward< Args >(args)...))
Definition: Container.hpp:433
mapped_type & operator[](key_type &&key)
Access the value that is mapped to a key equivalent to key, creating it if such key does not exist al...
Definition: Container.hpp:325
Self-contained description of a single IO operation.
Definition: IOTask.hpp:670
Logical compilation of data from one snapshot (e.g.
Definition: Iteration.hpp:127
Definition: ParticlePatches.hpp:32
Definition: ParticleSpecies.hpp:34
Implementation for the root level of the openPMD hierarchy.
Definition: Series.hpp:219
Definition: ReadIterations.hpp:35
Return an error string for read-only access.
Definition: OutOfRangeMsg.hpp:37
Definition: Attributable.hpp:57
Definition: Container.hpp:71
InternalContainer m_container
The wrapped container holding all the actual data, e.g.
Definition: Container.hpp:78
This class wraps a Container and forwards operator[]() and at() to it.
Definition: Container.hpp:532
void forget(K &&k)
Remove key from the list of accessed keys.
Definition: Container.hpp:579
Data members for Series.
Definition: Series.hpp:79
@ Parsing
All objects in the openPMD object model are temporarily mutable to allow inserting newly-parsed data.
Public definitions of openPMD-api.
Definition: Date.cpp:29
@ READ_ONLY
Open Series as read-only, fails if Series is not found.
Container Element Creation Policy.
Definition: Container.hpp:52