aqnwb 0.5.0
Loading...
Searching...
No Matches
HDF5IO.hpp
Go to the documentation of this file.
1#pragma once
2
3#include <iostream>
4#include <memory>
5#include <string>
6
7#include <H5Opublic.h>
8
9#include "Types.hpp"
10#include "io/BaseIO.hpp"
11#include "io/ReadIO.hpp"
12
13namespace H5
14{
15class DataSet;
16class Attribute;
17class H5File;
18class DataType;
19class Exception;
20class CommonFG;
21class PredType;
22class DataSpace;
23class FileAccPropList;
24} // namespace H5
25
30namespace AQNWB::IO::HDF5
31{
32
33class HDF5RecordingData; // forward declaration
34
61
66class HDF5IO : public BaseIO
67{
68public:
73 explicit HDF5IO(const std::string& fileName);
74
78 ~HDF5IO() override;
79
86
99
106 Status open() override;
107
115 Status open(FileMode mode) override;
116
129 Status open(FileMode mode, const ChunkCacheConfig& chunkCache);
130
131#ifdef H5_HAVE_ROS3_VFD
155 Status openS3(const std::string& aws_region = "",
156 const std::string& secret_id = "",
157 const std::string& secret_key = "",
158 const ChunkCacheConfig& chunkCache = ChunkCacheConfig());
159#endif // H5_HAVE_ROS3_VFD
160
161#ifdef AQNWB_HAVE_REMFILE_VFD
181 Status openRemote(const ChunkCacheConfig& chunkCache = ChunkCacheConfig());
182#endif // AQNWB_HAVE_REMFILE_VFD
183
188 Status close() override;
189
194 Status flush() override;
195
204 StorageObjectType getStorageObjectType(std::string path) const override;
205
224 std::string findObject(const std::string& name,
225 const std::string& starting_path = "/") const override;
226
240 AQNWB::IO::DataBlockGeneric readDataset(const std::string& dataPath,
241 const SizeArray& start = {},
242 const SizeArray& count = {},
243 const SizeArray& stride = {},
244 const SizeArray& block = {}) override;
245
260 const std::string& dataPath) const override;
261
268 std::string readReferenceAttribute(
269 const std::string& dataPath) const override;
270
281 const void* data,
282 const std::string& path,
283 const std::string& name,
284 const SizeType& size = 1) override;
285
294 Status createAttribute(const std::string& data,
295 const std::string& path,
296 const std::string& name,
297 const bool overwrite = false) override;
298
309 Status createAttribute(const std::vector<std::string>& data,
310 const std::string& path,
311 const std::string& name,
312 const bool overwrite = false) override;
313
321 Status createReferenceAttribute(const std::string& referencePath,
322 const std::string& path,
323 const std::string& name) override;
324
330 Status createGroup(const std::string& path) override;
331
341 Status createLink(const std::string& path,
342 const std::string& reference) override;
343
350 Status createStringDataSet(const std::string& path,
351 const std::string& value) override;
352
359 Status createStringDataSet(const std::string& path,
360 const std::vector<std::string>& values) override;
361
370 const std::string& path,
371 const std::vector<std::string>& references) override;
372
377 Status startRecording() override;
378
392 Status startRecording(bool disableSWMRMode);
393
398 Status stopRecording() override;
399
406 bool canModifyObjects() override;
407
416 std::unique_ptr<IO::BaseRecordingData> createArrayDataSet(
417 const IO::BaseArrayDataSetConfig& config,
418 const std::string& path) override;
419
425 SizeArray getStorageObjectShape(const std::string& path) const override;
426
434 SizeArray getStorageObjectChunking(const std::string& path) const override;
435
443 BaseDataType getStorageObjectDataType(const std::string& path) const override;
444
451 bool objectExists(const std::string& path) const override;
452
461 bool attributeExists(const std::string& path) const override;
462
481 virtual std::vector<std::pair<std::string, StorageObjectType>>
482 getStorageObjects(const std::string& path,
483 const StorageObjectType& objectType =
484 StorageObjectType::Undefined) const override;
485
492 H5O_type_t getH5ObjectType(const std::string& path) const;
493
504 static H5::DataType getNativeType(IO::BaseDataType type);
505
514 static IO::BaseDataType getBaseDataType(const H5::DataType& nativeType);
515
525 static H5::DataType getH5Type(IO::BaseDataType type);
526
527protected:
533 std::shared_ptr<IO::BaseRecordingData> getDataSetImpl(
534 const std::string& path) override;
535
541 Status createGroupIfDoesNotExist(const std::string& path) override;
542
543private:
553 static H5::FileAccPropList createFileAccessPropertyList(
554 const ChunkCacheConfig& chunkCache);
555
562 static bool isValidChunkCacheConfig(const ChunkCacheConfig& chunkCache);
563
574 static std::string fixedLengthString(const char* data, size_t size);
575
593 template<typename T, typename HDF5TYPE>
594 static std::vector<T> readDataHelper(const HDF5TYPE& dataSource,
595 size_t numElements,
596 const H5::DataSpace& memspace,
597 const H5::DataSpace& dataspace);
598
616 template<typename T, typename HDF5TYPE>
617 static std::vector<T> readDataHelper(const HDF5TYPE& dataSource,
618 size_t numElements);
619
632 template<typename HDF5TYPE>
633 static std::vector<std::string> readStringDataHelper(
634 const HDF5TYPE& dataSource,
635 size_t numElements,
636 const H5::DataSpace& memspace,
637 const H5::DataSpace& dataspace);
638
654 template<typename HDF5TYPE>
655 static std::vector<std::string> readStringDataHelper(
656 const HDF5TYPE& dataSource, size_t numElements);
657
658 std::unique_ptr<H5::Attribute> getAttribute(const std::string& path) const;
659
671
675 std::unique_ptr<H5::H5File> m_file;
676
682};
683
684} // namespace AQNWB::IO::HDF5
AQNWB::Types::StorageObjectType StorageObjectType
Definition BaseIO.hpp:20
AQNWB::Types::Status Status
Definition BaseIO.hpp:21
AQNWB::Types::SizeArray SizeArray
Definition BaseIO.hpp:22
AQNWB::Types::SizeType SizeType
Definition Channel.hpp:8
Base class for array dataset configuration.
Definition BaseIO.hpp:277
Represents a base data type.
Definition BaseIO.hpp:47
BaseIO(const std::string &filename)
Constructor for the BaseIO class.
Definition BaseIO.cpp:151
Generic structure to hold type-erased data and shape.
Definition ReadIO.hpp:34
Status stopRecording() override
Stops the recording process.
Definition HDF5IO.cpp:1417
SizeArray getStorageObjectChunking(const std::string &path) const override
Gets the chunking configuration of a dataset.
Definition HDF5IO.cpp:1558
std::string findObject(const std::string &name, const std::string &starting_path="/") const override
Search for the first object (Group or Dataset) whose last path component matches name,...
Definition HDF5IO.cpp:302
bool m_disableSWMRMode
Tracks whether SWMR mode is disabled for the current recording. Set by startRecording(bool) at the st...
Definition HDF5IO.hpp:681
Status createGroup(const std::string &path) override
Creates a new group in the file.
Definition HDF5IO.cpp:1245
static IO::BaseDataType getBaseDataType(const H5::DataType &nativeType)
Returns the BaseDataType for a given HDF5 native data type.
Definition HDF5IO.cpp:1836
Status createLink(const std::string &path, const std::string &reference) override
Creates a soft link to another location in the file.
Definition HDF5IO.cpp:1277
std::string readReferenceAttribute(const std::string &dataPath) const override
Reads a reference attribute and returns the path to the referenced object.
Definition HDF5IO.cpp:635
Status createReferenceDataSet(const std::string &path, const std::vector< std::string > &references) override
Creates a dataset that holds an array of references to groups within the file.
Definition HDF5IO.cpp:1297
bool attributeExists(const std::string &path) const override
Checks whether an Attribute exists at the location in the file.
Definition HDF5IO.cpp:1466
virtual std::vector< std::pair< std::string, StorageObjectType > > getStorageObjects(const std::string &path, const StorageObjectType &objectType=StorageObjectType::Undefined) const override
Gets the list of storage objects (groups, datasets, attributes) inside a group.
Definition HDF5IO.cpp:1473
BaseDataType getStorageObjectDataType(const std::string &path) const override
Gets the BaseDataType of a dataset or attribute.
Definition HDF5IO.cpp:1591
StorageObjectType getStorageObjectType(std::string path) const override
Get the storage type (Group, Dataset, Attribute) of the object at path.
Definition HDF5IO.cpp:469
static bool isValidChunkCacheConfig(const ChunkCacheConfig &chunkCache)
Checks the chunk cache configuration, printing an error if it is invalid.
Definition HDF5IO.cpp:57
AQNWB::IO::DataBlockGeneric readAttribute(const std::string &dataPath) const override
Reads a attribute and determines the data type.
Definition HDF5IO.cpp:698
static SizeType getDefaultChunkCacheNumBytes()
Get the default size of the raw data chunk cache (per dataset) of the HDF5 library in bytes,...
Definition HDF5IO.cpp:90
Status createAttribute(const IO::BaseDataType &type, const void *data, const std::string &path, const std::string &name, const SizeType &size=1) override
Creates an attribute at a given location in the file.
Definition HDF5IO.cpp:1014
Status startRecording() override
Start SWMR write to start recording process.
Definition HDF5IO.cpp:1396
std::unique_ptr< H5::Attribute > getAttribute(const std::string &path) const
Definition HDF5IO.cpp:436
SizeArray getStorageObjectShape(const std::string &path) const override
Returns the size of the dataset or attribute for each dimension.
Definition HDF5IO.cpp:1530
std::unique_ptr< IO::BaseRecordingData > createArrayDataSet(const IO::BaseArrayDataSetConfig &config, const std::string &path) override
Creates an extendable dataset with the given configuration and path.
Definition HDF5IO.cpp:1659
Status createReferenceAttribute(const std::string &referencePath, const std::string &path, const std::string &name) override
Sets an object reference attribute for a given location in the file.
Definition HDF5IO.cpp:1205
Status close() override
Closes the file.
Definition HDF5IO.cpp:260
bool canModifyObjects() override
Checks whether the file is in a mode where objects can be added or deleted. Note, this does not apply...
Definition HDF5IO.cpp:1432
HDF5IO(const std::string &fileName)
Constructor for the HDF5IO class that takes a file name as input.
Definition HDF5IO.cpp:34
static std::vector< T > readDataHelper(const HDF5TYPE &dataSource, size_t numElements, const H5::DataSpace &memspace, const H5::DataSpace &dataspace)
Reads data from an HDF5 dataset or attribute into a vector of the appropriate type.
Definition HDF5IO.cpp:497
SizeType getChunkCacheNumBytes() const
Get the size of the raw data chunk cache (per dataset) of the open file in bytes.
Definition HDF5IO.cpp:76
H5O_type_t getH5ObjectType(const std::string &path) const
Returns the HDF5 type of object at a given path.
Definition HDF5IO.cpp:1748
AQNWB::IO::DataBlockGeneric readDataset(const std::string &dataPath, const SizeArray &start={}, const SizeArray &count={}, const SizeArray &stride={}, const SizeArray &block={}) override
Reads a dataset or attribute and determines the data type.
Definition HDF5IO.cpp:830
Status createStringDataSet(const std::string &path, const std::string &value) override
Creates a non-modifiable dataset with a string value.
Definition HDF5IO.cpp:1344
Status openRemote(const ChunkCacheConfig &chunkCache=ChunkCacheConfig())
Opens a remote file over HTTP(S) in read-only mode using the remfile virtual file driver.
Definition HDF5IO.cpp:169
Status openS3(const std::string &aws_region="", const std::string &secret_id="", const std::string &secret_key="", const ChunkCacheConfig &chunkCache=ChunkCacheConfig())
Opens an existing file in read-only mode using the ROS3 driver.
Definition HDF5IO.cpp:120
static H5::DataType getH5Type(IO::BaseDataType type)
Returns the HDF5 data type for a given base data type.
Definition HDF5IO.cpp:1886
std::shared_ptr< IO::BaseRecordingData > getDataSetImpl(const std::string &path) override
Implementation of getDataSet to be provided by derived classes.
Definition HDF5IO.cpp:1636
bool objectExists(const std::string &path) const override
Checks whether a Dataset, Group, or Link already exists at the location in the file.
Definition HDF5IO.cpp:1456
Status closeFileImpl()
Non-virtual helper that performs the actual HDF5 file close.
Definition HDF5IO.cpp:266
static H5::DataType getNativeType(IO::BaseDataType type)
Returns the HDF5 native data type for a given base data type.
Definition HDF5IO.cpp:1778
static std::vector< std::string > readStringDataHelper(const HDF5TYPE &dataSource, size_t numElements, const H5::DataSpace &memspace, const H5::DataSpace &dataspace)
Reads a variable-length string from an HDF5 dataset or attribute.
Definition HDF5IO.cpp:529
Status open() override
Opens an existing file or creates a new file for writing.
Definition HDF5IO.cpp:198
Status flush() override
Flush data to disk.
Definition HDF5IO.cpp:283
static H5::FileAccPropList createFileAccessPropertyList(const ChunkCacheConfig &chunkCache)
Creates a new file access property list for opening a file.
Definition HDF5IO.cpp:40
Status createGroupIfDoesNotExist(const std::string &path) override
Creates a new group if it does not exist.
Definition HDF5IO.cpp:1263
~HDF5IO() override
Destructor.
Definition HDF5IO.cpp:102
std::unique_ptr< H5::H5File > m_file
Unique pointer to the HDF5 file for reading.
Definition HDF5IO.hpp:675
static std::string fixedLengthString(const char *data, size_t size)
Returns the value of a fixed-length string element.
Definition HDF5IO.cpp:70
Namespace for all components of the HDF5 I/O backend.
FileMode
The access mode for the file.
Definition BaseIO.hpp:245
Definition HDF5IO.hpp:14
Configuration of the HDF5 raw data chunk cache used for each dataset of a file (see H5Pset_cache),...
Definition HDF5IO.hpp:49
SizeType numBytes
Definition HDF5IO.hpp:52
double preemptionPolicy
Definition HDF5IO.hpp:59
SizeType numSlots
Definition HDF5IO.hpp:56