Series.hpp

/* Copyright 2017-2025 Fabian Koller, Axel Huebl, Franz Poeschel, Luca Fedeli
 *
 * This file is part of openPMD-api.
 *
 * openPMD-api is free software: you can redistribute it and/or modify
 * it under the terms of of either the GNU General Public License or
 * the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * openPMD-api is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License and the GNU Lesser General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License
 * and the GNU Lesser General Public License along with openPMD-api.
 * If not, see <http://www.gnu.org/licenses/>.
 */
#pragma once
 
#include "openPMD/Error.hpp"
#include "openPMD/IO/AbstractIOHandler.hpp"
#include "openPMD/IO/Access.hpp"
#include "openPMD/IO/Format.hpp"
#include "openPMD/Iteration.hpp"
#include "openPMD/IterationEncoding.hpp"
#include "openPMD/Streaming.hpp"
#include "openPMD/auxiliary/TypeTraits.hpp"
#include "openPMD/auxiliary/Variant.hpp"
#include "openPMD/backend/Attributable.hpp"
#include "openPMD/backend/Container.hpp"
#include "openPMD/backend/HierarchyVisitor.hpp"
#include "openPMD/backend/ParsePreference.hpp"
#include "openPMD/config.hpp"
#include "openPMD/snapshots/Snapshots.hpp"
#include "openPMD/version.hpp"
 
#if openPMD_HAVE_MPI
#include <mpi.h>
#endif
 
#include <cstdint> // uint64_t
#include <deque>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <stdexcept>
#include <string>
#include <tuple>
#include <unordered_map>
#include <variant>
#include <vector>
 
// expose private and protected members for invasive testing
#ifndef OPENPMD_private
#define OPENPMD_private private:
#endif
 
namespace openPMD
{
class ReadIterations;
class StatefulIterator;
class Series;
 
namespace internal
{
    /* Just a more self-documenting boolean used for
     * m_iterationEncodingSetExplicitly */
    enum class default_or_explicit : bool
    {
        default_,
        explicit_
    };
    class SeriesData final : public AttributableData
    {
    public:
        explicit SeriesData() = default;
 
        virtual ~SeriesData();
 
        SeriesData(SeriesData const &) = delete;
        SeriesData(SeriesData &&) = delete;
 
        SeriesData &operator=(SeriesData const &) = delete;
        SeriesData &operator=(SeriesData &&) = delete;
 
        using IterationIndex_t = Iteration::IterationIndex_t;
        using IterationsContainer_t = Iterations;
        Iterations iterations{};
 
        std::unique_ptr<StatefulIterator> m_sharedStatefulIterator;
        std::set<IterationIndex_t> m_currentlyActiveIterations;
        std::unordered_map<IterationIndex_t, size_t> m_snapshotToStep;
        std::unordered_map<IterationIndex_t, std::string> m_iterationFilenames;
        std::optional<std::string> m_overrideFilebasedFilename;
        std::string m_name;
        std::string m_filenamePrefix;
        std::string m_filenamePostfix;
        std::string m_filenameExtension;
        int m_filenamePadding = -1;
        IterationEncoding m_iterationEncoding{};
        /*
         * ADIOS2 should use variable-based encoding as default rather than
         * group-based encoding as much as possible.
         * Since this cannot be decided at construction time, groupBased
         * encoding is selected first, and re-decided later.
         * However, when group-based encoding is selected by the user explcitly,
         * that selection should not be changed again.
         * Hence, remember that here.
         */
        default_or_explicit m_iterationEncodingSetExplicitly =
            default_or_explicit::default_;
        Format m_format;
        StepStatus m_stepStatus = StepStatus::NoStep;
        bool m_parseLazily = false;
        uint64_t m_hintLazyParsingAfterTimeout = 20; // seconds
 
        bool m_wroteAtLeastOneIOStep = false;
 
        std::optional<ParsePreference> m_parsePreference;
 
        std::optional<std::function<AbstractIOHandler *(Series &)>>
            m_deferred_initialization = std::nullopt;
 
        void close();
 
#if openPMD_HAVE_MPI
        /*
         * @todo Once we have separate MPI headers, move this there.
         */
        std::optional<MPI_Comm> m_communicator;
#endif
 
        struct NoSourceSpecified
        {};
        struct SourceSpecifiedViaJSON
        {
            std::string value;
        };
        struct SourceSpecifiedManually
        {
            std::string value;
        };
 
        struct RankTableData
        {
            Attributable m_attributable;
            std::variant<
                NoSourceSpecified,
                SourceSpecifiedViaJSON,
                SourceSpecifiedManually>
                m_rankTableSource;
            std::optional<chunk_assignment::RankMeta> m_bufferedRead;
        };
        RankTableData m_rankTable;
    }; // SeriesData
 
    class SeriesInternal;
} // namespace internal
 
class Series : public Attributable
{
    friend class Attributable;
    friend class Iteration;
    friend class Writable;
    friend class ReadIterations;
    friend class StatefulIterator;
    friend class internal::SeriesData;
    friend class internal::AttributableData;
    friend class StatefulSnapshotsContainer;
    template <typename T, typename Series_type>
    friend T &internal::makeOwning(T &self, Series_type);
 
public:
    explicit Series();
 
#if openPMD_HAVE_MPI
    Series(
        std::string const &filepath,
        Access at,
        MPI_Comm comm,
        std::string const &options = "{}");
#endif
 
    Series(
        std::string const &filepath,
        Access at,
        std::string const &options = "{}");
 
    Series(Series const &) = default;
    Series(Series &&) = default;
 
    Series &operator=(Series const &) = default;
    Series &operator=(Series &&) = default;
 
    ~Series() override = default;
 
    using IterationIndex_t = Iteration::IterationIndex_t;
    using IterationsContainer_t = internal::SeriesData::IterationsContainer_t;
    Iterations iterations;
 
    operator bool() const;
 
    std::string openPMD() const;
    Series &setOpenPMD(std::string const &openPMD);
 
    uint32_t openPMDextension() const;
    Series &setOpenPMDextension(uint32_t openPMDextension);
 
    std::string basePath() const;
    Series &setBasePath(std::string const &basePath);
 
    std::string meshesPath() const;
    Series &setMeshesPath(std::string const &meshesPath);
 
    bool hasRankTableRead();
 
#if openPMD_HAVE_MPI
    chunk_assignment::RankMeta rankTable(bool collective);
#else
    chunk_assignment::RankMeta rankTable(bool collective = false);
#endif
 
    Series &setRankTable(std::string const &myRankInfo);
 
    std::string particlesPath() const;
    Series &setParticlesPath(std::string const &particlesPath);
 
    std::string author() const;
    Series &setAuthor(std::string const &author);
 
    std::string software() const;
    Series &setSoftware(
        std::string const &newName,
        std::string const &newVersion = std::string("unspecified"));
 
    std::string softwareVersion() const;
    [[deprecated(
        "Set the version with the second argument of setSoftware()")]] Series &
    setSoftwareVersion(std::string const &softwareVersion);
 
    std::string date() const;
    Series &setDate(std::string const &date);
 
    std::string softwareDependencies() const;
    Series &setSoftwareDependencies(std::string const &newSoftwareDependencies);
 
    std::string machine() const;
    Series &setMachine(std::string const &newMachine);
 
    IterationEncoding iterationEncoding() const;
    Series &setIterationEncoding(IterationEncoding iterationEncoding);
 
    std::string iterationFormat() const;
    Series &setIterationFormat(std::string const &iterationFormat);
 
    std::string name() const;
 
    Series &setName(std::string const &name);
 
    std::string backend() const;
    std::string backend();
 
    void flush(std::string backendConfig = "{}");
 
    ReadIterations readIterations();
 
    Snapshots snapshots();
 
    void parseBase();
 
    WriteIterations writeIterations();
 
    void close();
 
    [[nodiscard]] bool closed() const;
 
    void visitHierarchy(HierarchyVisitor &v, bool recursive) override;
 
    template <typename X = void, typename... Args>
    auto iterationFlush(Args &&...)
    {
        static_assert(
            auxiliary::dependent_false_v<X>,
            "Cannot call this on an instance of Series.");
    }
 
    // clang-format off
OPENPMD_private
    // clang-format on
 
    static constexpr char const *const BASEPATH = "/data/%T/";
 
    struct ParsedInput;
    using iterations_t = decltype(internal::SeriesData::iterations);
    using iterations_iterator = iterations_t::iterator;
 
    using Data_t = internal::SeriesData;
    std::shared_ptr<Data_t> m_series = nullptr;
 
    inline std::shared_ptr<Data_t> getShared()
    {
        return m_series;
    }
 
    inline Data_t &get()
    {
        if (m_series)
        {
            return *m_series;
        }
        else
        {
            throw std::runtime_error(
                "[Series] Cannot use default-constructed Series.");
        }
    }
 
    inline Data_t const &get() const
    {
        if (m_series)
        {
            return *m_series;
        }
        else
        {
            throw std::runtime_error(
                "[Series] Cannot use default-constructed Series.");
        }
    }
 
    inline void setData(std::shared_ptr<internal::SeriesData> series)
    {
        m_series = std::move(series);
        iterations = m_series->iterations;
        Attributable::setData(m_series);
    }
 
    std::unique_ptr<ParsedInput> parseInput(std::string);
    template <typename TracingJSON>
    void parseJsonOptions(TracingJSON &options, ParsedInput &);
    bool hasExpansionPattern(std::string filenameWithExtension);
    bool reparseExpansionPattern(std::string filenameWithExtension);
    template <typename... MPI_Communicator>
    void init(
        std::string const &filepath,
        Access at,
        std::string const &options,
        MPI_Communicator &&...);
    template <typename TracingJSON, typename... MPI_Communicator>
    std::tuple<std::unique_ptr<ParsedInput>, TracingJSON> initIOHandler(
        std::string const &filepath,
        std::string const &options,
        Access at,
        bool resolve_generic_extension,
        MPI_Communicator &&...);
    void initSeries(
        std::unique_ptr<AbstractIOHandler>, std::unique_ptr<ParsedInput>);
    void initDefaults(IterationEncoding, bool initAll = false);
    std::future<void> flush_impl(
        iterations_iterator begin,
        iterations_iterator end,
        internal::FlushParams const &flushParams,
        bool flushIOHandler = true);
    void flushFileBased(
        iterations_iterator begin,
        iterations_iterator end,
        internal::FlushParams const &flushParams,
        bool flushIOHandler = true);
    /*
     * Group-based and variable-based iteration layouts share a lot of logic
     * (realistically, the variable-based iteration layout only throws out
     *  one layer in the hierarchy).
     * As a convention, methods that deal with both layouts are called
     * .*GorVBased, short for .*GroupOrVariableBased
     */
    void flushGorVBased(
        iterations_iterator begin,
        iterations_iterator end,
        internal::FlushParams const &flushParams,
        bool flushIOHandler = true);
    void flushMeshesPath();
    void flushParticlesPath();
    void flushRankTable();
    /* Parameter `read_only_this_single_iteration` used for reopening an
     * Iteration after closing it.
     */
    void readFileBased(
        std::optional<IterationIndex_t> read_only_this_single_iteration);
    void readOneIterationFileBased(std::string const &filePath);
    std::vector<IterationIndex_t> readGorVBased(
        bool do_always_throw_errors,
        bool init,
        std::optional<IterationIndex_t> read_only_this_single_iteration);
    void readBase();
    std::string iterationFilename(IterationIndex_t i);
 
    enum class IterationOpened : bool
    {
        HasBeenOpened,
        RemainsClosed
    };
    /*
     * For use by flushFileBased, flushGorVBased
     * Open an iteration, but only if necessary.
     * Only open if the iteration is dirty and if it is not in deferred
     * parse state.
     */
    IterationOpened
    openIterationIfDirty(IterationIndex_t index, Iteration &iteration);
    /*
     * Open an iteration. Ensures that the iteration's m_closed status
     * is set properly and that any files pertaining to the iteration
     * is opened.
     * Does not create files when called in CREATE mode.
     */
    void openIteration(IterationIndex_t index, Iteration &iteration);
 
    iterations_iterator indexOf(Iteration const &);
 
    AdvanceStatus advance(
        AdvanceMode mode,
        internal::AttributableData &file,
        iterations_iterator it);
 
    AdvanceStatus advance(AdvanceMode mode);
 
    void flushStep(bool doFlush);
 
    /*
     * setIterationEncoding() should only be called by users of our public API,
     * but never internally. We need to distinguish if the iteration encoding
     * was selected explicitly or implicitly, see
     * m_iterationEncodingSetExplicitly for further details.
     */
    Series &setIterationEncoding_internal(
        IterationEncoding iterationEncoding, internal::default_or_explicit);
 
    /*
     * Returns the current content of the /data/snapshot attribute.
     * (We could also add this to the public API some time)
     */
    std::optional<std::vector<IterationIndex_t>> currentSnapshot();
 
    AbstractIOHandler *runDeferredInitialization();
 
    AbstractIOHandler *IOHandler();
    AbstractIOHandler const *IOHandler() const;
 
    /* adios2::Mode::ReadRandomAccess does not support reading modifiable
     * attributes. However, we need the values of /data/snapshot as a modifiable
     * attribute, so this function quickly opens the file in adios2::Mode::Read
     * and retrieves the changings values over time.
     * Return std::nullopt if /data/snapshot is not present.
     */
    std::optional<std::vector<std::vector<IterationIndex_t>>>
    preparseSnapshots();
 
    Snapshots makeRandomAccessSnapshots();
    Snapshots makeSynchronousSnapshots();
    /* Should adios2::Variable<T>::SetStepSelection() be used for accessing
     * steps?
     */
    [[nodiscard]] bool randomAccessSteps() const;
 
    std::vector<std::string> availableDatasets();
}; // Series
 
namespace debug
{
    void printDirty(Series const &);
}
} // namespace openPMD
 
// Make sure that this legacy header is always included if Series.hpp is
// included, otherwise Series::readIterations() cannot be used
#include "openPMD/ReadIterations.hpp"