openPMD-api
RecordComponent.hpp
1 /* Copyright 2017-2020 Fabian Koller
2  *
3  * This file is part of openPMD-api.
4  *
5  * openPMD-api is free software: you can redistribute it and/or modify
6  * it under the terms of of either the GNU General Public License or
7  * the GNU Lesser General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * openPMD-api is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License and the GNU Lesser General Public License
15  * for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * and the GNU Lesser General Public License along with openPMD-api.
19  * If not, see <http://www.gnu.org/licenses/>.
20  */
21 #pragma once
22 
23 #include "openPMD/backend/BaseRecordComponent.hpp"
24 #include "openPMD/auxiliary/ShareRaw.hpp"
25 #include "openPMD/Dataset.hpp"
26 
27 #include <cmath>
28 #include <memory>
29 #include <limits>
30 #include <queue>
31 #include <string>
32 #include <sstream>
33 #include <stdexcept>
34 #include <type_traits>
35 #include <vector>
36 #include <array>
37 
38 // expose private and protected members for invasive testing
39 #ifndef OPENPMD_protected
40 # define OPENPMD_protected protected
41 #endif
42 
43 
44 namespace openPMD
45 {
46 namespace traits
47 {
56 template< typename T >
58 {
59  static constexpr bool value = false;
60 };
61 
62 template< typename T_Value >
63 struct IsContiguousContainer< std::vector< T_Value > >
64 {
65  static constexpr bool value = true;
66 };
67 
68 template<
69  typename T_Value,
70  std::size_t N
71 >
72 struct IsContiguousContainer< std::array< T_Value, N > >
73 {
74  static constexpr bool value = true;
75 };
76 } // namespace traits
77 
79 {
80  template<
81  typename T,
82  typename T_key,
83  typename T_container
84  >
85  friend class Container;
86  friend class Iteration;
87  friend class ParticleSpecies;
88  template< typename T_elem >
89  friend class BaseRecord;
90  friend class Record;
91  friend class Mesh;
92 
93 public:
94  enum class Allocation
95  {
96  USER,
97  API,
98  AUTO
99  }; // Allocation
100 
101  RecordComponent& setUnitSI(double);
102 
103  RecordComponent& resetDataset(Dataset);
104 
105  uint8_t getDimensionality() const;
106  Extent getExtent() const;
107 
116  template< typename T >
117  RecordComponent& makeConstant(T);
118 
127  template< typename T >
128  RecordComponent& makeEmpty( uint8_t dimensions );
129 
137  bool empty() const;
138 
146  template< typename T >
147  std::shared_ptr< T > loadChunk(
148  Offset = {0u},
149  Extent = {-1u},
150  double targetUnitSI = std::numeric_limits< double >::quiet_NaN() );
151 
161  template< typename T >
162  void loadChunk(
163  std::shared_ptr< T >,
164  Offset,
165  Extent,
166  double targetUnitSI = std::numeric_limits< double >::quiet_NaN() );
167 
168  template< typename T >
169  void storeChunk(std::shared_ptr< T >, Offset, Extent);
170 
171  template< typename T_ContiguousContainer >
172  typename std::enable_if<
174  >::type
175  storeChunk(T_ContiguousContainer &, Offset = {0u}, Extent = {-1u});
176 
177  static constexpr char const * const SCALAR = "\vScalar";
178 
179  virtual ~RecordComponent() = default;
180 
181 OPENPMD_protected:
182  RecordComponent();
183 
184  void readBase();
185 
186  std::shared_ptr< std::queue< IOTask > > m_chunks;
187  std::shared_ptr< Attribute > m_constantValue;
188  std::shared_ptr< bool > m_isEmpty = std::make_shared< bool >( false );
189 
190 private:
191  void flush(std::string const&);
192  virtual void read();
193 
200  RecordComponent& makeEmpty( Dataset d );
201 
210  bool
211  dirtyRecursive() const;
212 }; // RecordComponent
213 
214 
215 template< typename T >
216 inline RecordComponent&
218 {
219  if( written() )
220  throw std::runtime_error("A recordComponent can not (yet) be made constant after it has been written.");
221 
222  *m_constantValue = Attribute(value);
223  *m_isConstant = true;
224  return *this;
225 }
226 
227 template< typename T >
228 inline RecordComponent&
229 RecordComponent::makeEmpty( uint8_t dimensions )
230 {
231  return makeEmpty( Dataset(
232  determineDatatype< T >(),
233  Extent( dimensions, 0 ) ) );
234 }
235 
236 template< typename T >
237 inline std::shared_ptr< T >
238 RecordComponent::loadChunk(Offset o, Extent e, double targetUnitSI)
239 {
240  uint8_t dim = getDimensionality();
241 
242  // default arguments
243  // offset = {0u}: expand to right dim {0u, 0u, ...}
244  Offset offset = o;
245  if( o.size() == 1u && o.at(0) == 0u && dim > 1u )
246  offset = Offset(dim, 0u);
247 
248  // extent = {-1u}: take full size
249  Extent extent(dim, 1u);
250  if( e.size() == 1u && e.at(0) == -1u )
251  {
252  extent = getExtent();
253  for( uint8_t i = 0u; i < dim; ++i )
254  extent[i] -= offset[i];
255  }
256  else
257  extent = e;
258 
259  uint64_t numPoints = 1u;
260  for( auto const& dimensionSize : extent )
261  numPoints *= dimensionSize;
262 
263  auto newData = std::shared_ptr<T>(new T[numPoints], []( T *p ){ delete [] p; });
264  loadChunk(newData, offset, extent, targetUnitSI);
265  return newData;
266 }
267 
268 template< typename T >
269 inline void
270 RecordComponent::loadChunk(std::shared_ptr< T > data, Offset o, Extent e, double targetUnitSI)
271 {
272  if( !std::isnan(targetUnitSI) )
273  throw std::runtime_error("unitSI scaling during chunk loading not yet implemented");
274  Datatype dtype = determineDatatype(data);
275  if( dtype != getDatatype() )
276  if( !isSameInteger< T >( dtype ) &&
277  !isSameFloatingPoint< T >( dtype ) &&
278  !isSameComplexFloatingPoint< T >( dtype ) )
279  throw std::runtime_error("Type conversion during chunk loading not yet implemented");
280 
281  uint8_t dim = getDimensionality();
282 
283  // default arguments
284  // offset = {0u}: expand to right dim {0u, 0u, ...}
285  Offset offset = o;
286  if( o.size() == 1u && o.at(0) == 0u && dim > 1u )
287  offset = Offset(dim, 0u);
288 
289  // extent = {-1u}: take full size
290  Extent extent(dim, 1u);
291  if( e.size() == 1u && e.at(0) == -1u )
292  {
293  extent = getExtent();
294  for( uint8_t i = 0u; i < dim; ++i )
295  extent[i] -= offset[i];
296  }
297  else
298  extent = e;
299 
300  if( extent.size() != dim || offset.size() != dim )
301  {
302  std::ostringstream oss;
303  oss << "Dimensionality of chunk ("
304  << "offset=" << offset.size() << "D, "
305  << "extent=" << extent.size() << "D) "
306  << "and record component ("
307  << int(dim) << "D) "
308  << "do not match.";
309  throw std::runtime_error(oss.str());
310  }
311  Extent dse = getExtent();
312  for( uint8_t i = 0; i < dim; ++i )
313  if( dse[i] < offset[i] + extent[i] )
314  throw std::runtime_error("Chunk does not reside inside dataset (Dimension on index " + std::to_string(i)
315  + " - DS: " + std::to_string(dse[i])
316  + " - Chunk: " + std::to_string(offset[i] + extent[i])
317  + ")");
318  if( !data )
319  throw std::runtime_error("Unallocated pointer passed during chunk loading.");
320 
321  if( constant() )
322  {
323  uint64_t numPoints = 1u;
324  for( auto const& dimensionSize : extent )
325  numPoints *= dimensionSize;
326 
327  T value = m_constantValue->get< T >();
328 
329  T* raw_ptr = data.get();
330  std::fill(raw_ptr, raw_ptr + numPoints, value);
331  } else
332  {
334  dRead.offset = offset;
335  dRead.extent = extent;
336  dRead.dtype = getDatatype();
337  dRead.data = std::static_pointer_cast< void >(data);
338  m_chunks->push(IOTask(this, dRead));
339  }
340 }
341 
342 template< typename T >
343 inline void
344 RecordComponent::storeChunk(std::shared_ptr<T> data, Offset o, Extent e)
345 {
346  if( constant() )
347  throw std::runtime_error("Chunks cannot be written for a constant RecordComponent.");
348  if( empty() )
349  throw std::runtime_error("Chunks cannot be written for an empty RecordComponent.");
350  if( !data )
351  throw std::runtime_error("Unallocated pointer passed during chunk store.");
352  Datatype dtype = determineDatatype(data);
353  if( dtype != getDatatype() )
354  {
355  std::ostringstream oss;
356  oss << "Datatypes of chunk data ("
357  << dtype
358  << ") and record component ("
359  << getDatatype()
360  << ") do not match.";
361  throw std::runtime_error(oss.str());
362  }
363  uint8_t dim = getDimensionality();
364  if( e.size() != dim || o.size() != dim )
365  {
366  std::ostringstream oss;
367  oss << "Dimensionality of chunk ("
368  << "offset=" << o.size() << "D, "
369  << "extent=" << e.size() << "D) "
370  << "and record component ("
371  << int(dim) << "D) "
372  << "do not match.";
373  throw std::runtime_error(oss.str());
374  }
375  Extent dse = getExtent();
376  for( uint8_t i = 0; i < dim; ++i )
377  if( dse[i] < o[i] + e[i] )
378  throw std::runtime_error("Chunk does not reside inside dataset (Dimension on index " + std::to_string(i)
379  + " - DS: " + std::to_string(dse[i])
380  + " - Chunk: " + std::to_string(o[i] + e[i])
381  + ")");
382 
384  dWrite.offset = o;
385  dWrite.extent = e;
386  dWrite.dtype = dtype;
387  /* std::static_pointer_cast correctly reference-counts the pointer */
388  dWrite.data = std::static_pointer_cast< void const >(data);
389  m_chunks->push(IOTask(this, dWrite));
390 }
391 
392 template< typename T_ContiguousContainer >
393 inline typename std::enable_if<
395 >::type
396 RecordComponent::storeChunk(T_ContiguousContainer & data, Offset o, Extent e)
397 {
398  uint8_t dim = getDimensionality();
399 
400  // default arguments
401  // offset = {0u}: expand to right dim {0u, 0u, ...}
402  Offset offset = o;
403  if( o.size() == 1u && o.at(0) == 0u && dim > 1u )
404  offset = Offset(dim, 0u);
405 
406  // extent = {-1u}: take full size
407  Extent extent(dim, 1u);
408  // avoid outsmarting the user:
409  // - stdlib data container implement 1D -> 1D chunk to write
410  if( e.size() == 1u && e.at(0) == -1u && dim == 1u )
411  extent.at(0) = data.size();
412  else
413  extent = e;
414 
415  storeChunk(shareRaw(data), offset, extent);
416 }
417 } // namespace openPMD
Definition: Dataset.hpp:36
Self-contained description of a single IO operation.
Definition: IOTask.hpp:471
RecordComponent & makeEmpty(uint8_t dimensions)
Create a dataset with zero extent in each dimension.
Definition: RecordComponent.hpp:229
Logical compilation of data from one snapshot (e.g.
Definition: Iteration.hpp:35
Emulate in the C++17 concept ContiguousContainer.
Definition: RecordComponent.hpp:57
RecordComponent & makeConstant(T)
Create a dataset with regular extent and constant value.
Definition: RecordComponent.hpp:217
STL namespace.
std::shared_ptr< T > shareRaw(T *x)
Share ownership with a raw pointer.
Definition: ShareRaw.hpp:47
Datatype
Concrete datatype of an object available at runtime.
Definition: Datatype.hpp:42
amount of substance
Varidic datatype supporting at least all formats for attributes specified in the openPMD standard...
Definition: Attribute.hpp:50
Definition: RecordComponent.hpp:78
Public definitions of openPMD-api.
Definition: Date.cpp:28
Definition: Record.hpp:33
Definition: BaseRecord.hpp:36
Definition: ParticleSpecies.hpp:34
Container for N-dimensional, homogeneous Records.
Definition: Mesh.hpp:39
Map-like container that enforces openPMD requirements and handles IO.
Definition: Container.hpp:70
std::shared_ptr< T > loadChunk(Offset={0u}, Extent={-1u}, double targetUnitSI=std::numeric_limits< double >::quiet_NaN())
Load and allocate a chunk of data.
Definition: RecordComponent.hpp:238
Definition: BaseRecordComponent.hpp:34