areaDetector  3-13
EPICS areaDetector framework
NDFileHDF5.h
Go to the documentation of this file.
1 /* NDFileHDF5.h
2  * Writes NDArrays to HDF5 files.
3  *
4  * Ulrik Kofoed Pedersen
5  * March 20. 2011
6  */
7 #ifndef NDFileHDF5_H
8 #define NDFileHDF5_H
9 
10 #include <list>
11 #include <string.h>
12 #include <hdf5.h>
13 #include <NDPluginFile.h>
14 #include <NDArray.h>
15 #include "NDFileHDF5Layout.h"
16 #include "NDFileHDF5Dataset.h"
17 #include "NDFileHDF5LayoutXML.h"
19 #include "NDFileHDF5VersionCheck.h"
20 #include "Codec.h"
21 
22 #define MAXEXTRADIMS 10
23 #define MAX_CHUNK_DIMS ND_ARRAY_MAX_DIMS
24 
25 #define str_NDFileHDF5_chunkSizeAuto "HDF5_chunkSizeAuto"
26 #define str_NDFileHDF5_nFramesChunks "HDF5_nFramesChunks"
27 #define str_NDFileHDF5_chunkBoundaryAlign "HDF5_chunkBoundaryAlign"
28 #define str_NDFileHDF5_chunkBoundaryThreshold "HDF5_chunkBoundaryThreshold"
29 #define str_NDFileHDF5_NDAttributeChunk "HDF5_NDAttributeChunk"
30 #define str_NDFileHDF5_nExtraDims "HDF5_nExtraDims"
31 #define str_NDFileHDF5_extraDimOffsetX "HDF5_extraDimOffsetX"
32 #define str_NDFileHDF5_extraDimOffsetY "HDF5_extraDimOffsetY"
33 #define str_NDFileHDF5_storeAttributes "HDF5_storeAttributes"
34 #define str_NDFileHDF5_storePerformance "HDF5_storePerformance"
35 #define str_NDFileHDF5_totalRuntime "HDF5_totalRuntime"
36 #define str_NDFileHDF5_totalIoSpeed "HDF5_totalIoSpeed"
37 #define str_NDFileHDF5_flushNthFrame "HDF5_flushNthFrame"
38 #define str_NDFileHDF5_compressionType "HDF5_compressionType"
39 #define str_NDFileHDF5_nbitsPrecision "HDF5_nbitsPrecision"
40 #define str_NDFileHDF5_nbitsOffset "HDF5_nbitsOffset"
41 #define str_NDFileHDF5_szipNumPixels "HDF5_szipNumPixels"
42 #define str_NDFileHDF5_zCompressLevel "HDF5_zCompressLevel"
43 #define str_NDFileHDF5_bloscShuffleType "HDF5_bloscShuffleType"
44 #define str_NDFileHDF5_bloscCompressor "HDF5_bloscCompressor"
45 #define str_NDFileHDF5_bloscCompressLevel "HDF5_bloscCompressLevel"
46 #define str_NDFileHDF5_jpegQuality "HDF5_jpegQuality"
47 #define str_NDFileHDF5_dimAttDatasets "HDF5_dimAttDatasets"
48 #define str_NDFileHDF5_layoutErrorMsg "HDF5_layoutErrorMsg"
49 #define str_NDFileHDF5_layoutValid "HDF5_layoutValid"
50 #define str_NDFileHDF5_layoutFilename "HDF5_layoutFilename"
51 #define str_NDFileHDF5_posRunning "HDF5_posRunning"
52 #define str_NDFileHDF5_posNameDimN "HDF5_posNameDimN"
53 #define str_NDFileHDF5_posNameDimX "HDF5_posNameDimX"
54 #define str_NDFileHDF5_posNameDimY "HDF5_posNameDimY"
55 #define str_NDFileHDF5_posIndexDimN "HDF5_posIndexDimN"
56 #define str_NDFileHDF5_posIndexDimX "HDF5_posIndexDimX"
57 #define str_NDFileHDF5_posIndexDimY "HDF5_posIndexDimY"
58 #define str_NDFileHDF5_fillValue "HDF5_fillValue"
59 #define str_NDFileHDF5_SWMRFlushNow "HDF5_SWMRFlushNow"
60 #define str_NDFileHDF5_SWMRCbCounter "HDF5_SWMRCbCounter"
61 #define str_NDFileHDF5_SWMRSupported "HDF5_SWMRSupported"
62 #define str_NDFileHDF5_SWMRMode "HDF5_SWMRMode"
63 #define str_NDFileHDF5_SWMRRunning "HDF5_SWMRRunning"
64 
68 {
69  public:
70  static const char *str_NDFileHDF5_chunkSize[MAX_CHUNK_DIMS];
71  static const char *str_NDFileHDF5_extraDimSize[MAXEXTRADIMS];
72  static const char *str_NDFileHDF5_extraDimName[MAXEXTRADIMS];
73  static const char *str_NDFileHDF5_extraDimChunk[MAXEXTRADIMS];
74  static const char *str_NDFileHDF5_posName[MAXEXTRADIMS];
75  static const char *str_NDFileHDF5_posIndex[MAXEXTRADIMS];
76 
77  NDFileHDF5(const char *portName, int queueSize, int blockingCallbacks,
78  const char *NDArrayPort, int NDArrayAddr,
79  int priority, int stackSize);
80 
81  /* The methods that this class implements */
82  virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray);
83  virtual asynStatus readFile(NDArray **pArray);
84  virtual asynStatus writeFile(NDArray *pArray);
85  virtual asynStatus closeFile();
86  virtual void report(FILE *fp, int details);
87  virtual asynStatus writeInt32(asynUser *pasynUser, epicsInt32 value);
88  virtual asynStatus writeOctet(asynUser *pasynUser, const char *value, size_t nChars, size_t *nActual);
89 
90  void flushTask();
91  asynStatus startSWMR();
92  asynStatus flushCallback();
93  asynStatus createXMLFileLayout();
94  asynStatus storeOnOpenAttributes();
95  asynStatus storeOnCloseAttributes();
96  asynStatus storeOnOpenCloseAttribute(hdf5::Element *element, bool open);
97  asynStatus createTree(hdf5::Group* root, hid_t h5handle);
98  asynStatus createHardLinks(hdf5::Group* root);
99 
100  hid_t writeHdfConstDataset( hid_t h5_handle, hdf5::Dataset* dset);
101  hid_t writeH5dsetStr(hid_t element, const std::string &name, const std::string &str_value) const;
102  hid_t writeH5dsetInt32(hid_t element, const std::string &name, const std::string &str_value) const;
103  hid_t writeH5dsetFloat64(hid_t element, const std::string &name, const std::string &str_value) const;
104 
105  void writeHdfAttributes( hid_t h5_handle, hdf5::Element* element);
106  hid_t createDataset(hid_t group, hdf5::Dataset *dset);
107  void writeH5attrStr(hid_t element,
108  const std::string &attr_name,
109  const std::string &str_attr_value) const;
110  void writeH5attrInt32(hid_t element, const std::string &attr_name, const std::string &str_attr_value) const;
111  void writeH5attrFloat64(hid_t element, const std::string &attr_name, const std::string &str_attr_value) const;
112  hid_t fromHdfToHidDatatype(hdf5::DataType_t in) const;
113  hid_t createDatasetDetector(hid_t group, hdf5::Dataset *dset);
114 
115  int fileExists(char *filename);
116  int verifyLayoutXMLFile();
117 
118  hsize_t getDim(int index);
119  hsize_t getMaxDim(int index);
120  hsize_t getChunkDim(int index);
121  hsize_t getOffset(int index);
122  hsize_t getVirtualDim(int index);
123 
124  std::map<std::string, NDFileHDF5Dataset *> detDataMap; // Map of handles to detector datasets, indexed by name
125  std::map<std::string, hid_t> constDsetMap; // Map of handles to constant datasets, indexed by name
126  std::string defDsetName; // Name of the default data set
127  std::string ndDsetName; // Name of NDAttribute that specifies the destination data set
128  std::map<std::string, hdf5::Element *> onOpenMap; // Map of handles to elements with onOpen ndattributes, indexed by fullname
129  std::map<std::string, hdf5::Element *> onCloseMap; // Map of handles to elements with onClose ndattributes, indexed by fullname
130  Codec_t codec; // Definition of codec used to compress the data.
131 
132  protected:
133  /* plugin parameters */
135  #define FIRST_NDFILE_HDF5_PARAM NDFileHDF5_chunkSizeAuto
137  int NDFileHDF5_chunkSize[MAX_CHUNK_DIMS];
144  int NDFileHDF5_extraDimSize[MAXEXTRADIMS];
145  int NDFileHDF5_extraDimName[MAXEXTRADIMS];
146  int NDFileHDF5_extraDimChunk[MAXEXTRADIMS];
166  int NDFileHDF5_posName[MAXEXTRADIMS];
167  int NDFileHDF5_posIndex[MAXEXTRADIMS];
174 
175  asynStatus configureDims(NDArray *pArray);
176  void calcNumFrames();
177  void setMultiFrameFile(bool multi);
178 
179  private:
180  /* private helper functions */
181  inline bool IsPrime(int number)
182  {
183  if (((!(number & 1)) && number != 2) || (number < 2) || (number % 3 == 0 && number != 3)){
184  return false;
185  }
186 
187  for(int k = 1; 36*k*k-12*k < number;++k){
188  if ((number % (6*k+1) == 0) || (number % (6*k-1) == 0)){
189  return false;
190  }
191  }
192  return true;
193  };
194 
195  hid_t typeNd2Hdf(NDDataType_t datatype);
196  asynStatus configureDatasetDims(NDArray *pArray);
197  asynStatus configureDatasetCompression();
198  asynStatus configureCompression(NDArray *pArray);
199  char* getDimsReport();
200  asynStatus writeStringAttribute(hid_t element, const char* attrName, const char* attrStrValue);
201  asynStatus calculateAttributeChunking(int *chunking, int *mdim_chunking);
202  asynStatus writeAttributeDataset(hdf5::When_t whenToSave, int positionMode, hsize_t *offsets);
203  asynStatus closeAttributeDataset();
204  asynStatus configurePerformanceDataset();
205  asynStatus createPerformanceDataset();
206  asynStatus writePerformanceDataset();
207  unsigned int calcIstorek();
208  hsize_t calcChunkCacheBytes();
209  hsize_t calcChunkCacheSlots();
210 
211  void checkForOpenFile();
212  bool checkForSWMRMode();
213  bool checkForSWMRSupported();
214  void addDefaultAttributes(NDArray *pArray);
215  asynStatus writeDefaultDatasetAttributes(NDArray *pArray);
216  asynStatus createNewFile(const char *fileName);
217  asynStatus createFileLayout(NDArray *pArray);
218  asynStatus createAttributeDataset(NDArray *pArray);
219  int isAttributeIndex(const std::string& attName);
220  epicsInt32 findPositionIndex(NDArray *pArray, char *posName);
221 
222 
223  hdf5::LayoutXML layout;
224 
225  int nextRecord;
226  int *pAttributeId;
227  NDAttributeList *pFileAttributes;
228  bool multiFrameFile;
229  char *extraDimNameN;
230  char *extraDimNameX;
231  char *extraDimNameY;
232  char *extraDimName[MAXEXTRADIMS];
233  double *performanceBuf;
234  double *performancePtr;
235  epicsInt32 numPerformancePoints;
236  epicsTimeStamp prevts;
237  epicsTimeStamp opents;
238  epicsTimeStamp firstFrame;
239  double frameSize;
240  int bytesPerElement;
241  char *hostname;
242 
243  epicsEventId flushEventId;
244  epicsMutex flushLock;
245 
246  std::list<NDFileHDF5AttributeDataset*> attrList;
247 
248  /* HDF5 handles and references */
249  hid_t file;
250  hid_t dataspace;
251  hid_t datatype;
252  hid_t cparms;
253  void *ptrFillValue;
254  hid_t perf_dataset_id;
255 
256  /* dimension descriptors */
257  int rank;
258  int nvirtual;
259  hsize_t *dims;
260  hsize_t *maxdims;
261  hsize_t *chunkdims;
262  hsize_t *offset;
263  hsize_t *framesize;
264  hsize_t *virtualdims;
265  char *ptrDimensionNames[ND_ARRAY_MAX_DIMS + MAXEXTRADIMS];
267  char *dimsreport;
268 };
269 
270 #endif
271 
#define ND_ARRAY_MAX_DIMS
NDArray.h.
Definition: NDArray.h:26
NDDataType_t
Enumeration of NDArray data types.
Definition: NDAttribute.h:30
#define MAX_CHUNK_DIMS
Definition: NDFileHDF5.h:23
#define MAXEXTRADIMS
Definition: NDFileHDF5.h:22
#define NDPLUGIN_API
Definition: NDPluginAPI.h:41
int NDFileOpenMode_t
Definition: NDPluginFile.h:17
N-dimensional array class; each array has a set of dimensions, a data type, pointer to data,...
Definition: NDArray.h:99
NDAttributeList.h.
Definition: NDAttributeList.h:21
Writes NDArrays in the HDF5 file format; an XML file can control the structure of the HDF5 file.
Definition: NDFileHDF5.h:68
int NDFileHDF5_nExtraDims
Definition: NDFileHDF5.h:141
int NDFileHDF5_dimAttDatasets
Definition: NDFileHDF5.h:161
int NDFileHDF5_extraDimOffsetY
Definition: NDFileHDF5.h:143
int NDFileHDF5_nbitsOffset
Definition: NDFileHDF5.h:154
int NDFileHDF5_SWMRCbCounter
Definition: NDFileHDF5.h:170
int NDFileHDF5_flushNthFrame
Definition: NDFileHDF5.h:151
std::map< std::string, hdf5::Element * > onOpenMap
Definition: NDFileHDF5.h:128
int NDFileHDF5_storePerformance
Definition: NDFileHDF5.h:148
int NDFileHDF5_zCompressLevel
Definition: NDFileHDF5.h:156
int NDFileHDF5_layoutFilename
Definition: NDFileHDF5.h:164
int NDFileHDF5_fillValue
Definition: NDFileHDF5.h:168
int NDFileHDF5_SWMRSupported
Definition: NDFileHDF5.h:171
Codec_t codec
Definition: NDFileHDF5.h:130
int NDFileHDF5_jpegQuality
Definition: NDFileHDF5.h:160
int NDFileHDF5_bloscCompressLevel
Definition: NDFileHDF5.h:158
int NDFileHDF5_extraDimOffsetX
Definition: NDFileHDF5.h:142
int NDFileHDF5_totalIoSpeed
Definition: NDFileHDF5.h:150
int NDFileHDF5_bloscCompressor
Definition: NDFileHDF5.h:157
int NDFileHDF5_SWMRMode
Definition: NDFileHDF5.h:172
int NDFileHDF5_szipNumPixels
Definition: NDFileHDF5.h:155
int NDFileHDF5_layoutValid
Definition: NDFileHDF5.h:163
int NDFileHDF5_bloscShuffleType
Definition: NDFileHDF5.h:159
int NDFileHDF5_NDAttributeChunk
Definition: NDFileHDF5.h:140
int NDFileHDF5_chunkSizeAuto
Definition: NDFileHDF5.h:134
std::map< std::string, hid_t > constDsetMap
Definition: NDFileHDF5.h:125
int NDFileHDF5_storeAttributes
Definition: NDFileHDF5.h:147
std::string defDsetName
Definition: NDFileHDF5.h:126
int NDFileHDF5_posRunning
Definition: NDFileHDF5.h:165
int NDFileHDF5_nFramesChunks
Definition: NDFileHDF5.h:136
std::map< std::string, hdf5::Element * > onCloseMap
Definition: NDFileHDF5.h:129
std::map< std::string, NDFileHDF5Dataset * > detDataMap
Definition: NDFileHDF5.h:124
int NDFileHDF5_chunkBoundaryAlign
Definition: NDFileHDF5.h:138
int NDFileHDF5_SWMRFlushNow
Definition: NDFileHDF5.h:169
int NDFileHDF5_SWMRRunning
Definition: NDFileHDF5.h:173
int NDFileHDF5_totalRuntime
Definition: NDFileHDF5.h:149
int NDFileHDF5_chunkBoundaryThreshold
Definition: NDFileHDF5.h:139
std::string ndDsetName
Definition: NDFileHDF5.h:127
int NDFileHDF5_layoutErrorMsg
Definition: NDFileHDF5.h:162
int NDFileHDF5_compressionType
Definition: NDFileHDF5.h:152
int NDFileHDF5_nbitsPrecision
Definition: NDFileHDF5.h:153
virtual asynStatus writeOctet(asynUser *pasynUser, const char *value, size_t maxChars, size_t *nActual)
Called when asyn clients call pasynOctet->write().
Definition: NDPluginDriver.cpp:804
Base class for NDArray file writing plugins; actual file writing plugins inherit from this class.
Definition: NDPluginFile.h:28
virtual asynStatus readFile(NDArray **pArray)=0
Read NDArray data from a file; pure virtual function that must be implemented by derived classes.
virtual asynStatus closeFile()=0
Close the file opened with NDPluginFile::openFile; pure virtual function that must be implemented by ...
virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray)=0
Open a file; pure virtual function that must be implemented by derived classes.
virtual asynStatus writeFile(NDArray *pArray)=0
Write NDArray data to a file; pure virtual function that must be implemented by derived classes.
virtual asynStatus writeInt32(asynUser *pasynUser, epicsInt32 value)
Called when asyn clients call pasynInt32->write().
Definition: NDPluginFile.cpp:799
virtual void report(FILE *fp, int details)
Report status of the driver.
Definition: asynNDArrayDriver.cpp:753
Class used for writing a DataSet with the NDFileHDF5 plugin.
Definition: NDFileHDF5Layout.h:149
Describe a generic structure element with the NDFileHDF5 plugin.
Definition: NDFileHDF5Layout.h:115
Describe a group element.
Definition: NDFileHDF5Layout.h:221
Used to define layout of HDF5 file with NDFileHDF5 plugin.
Definition: NDFileHDF5LayoutXML.h:62
When_t
Definition: NDFileHDF5Layout.h:20
DataType_t
Definition: NDFileHDF5Layout.h:33
@ string
Definition: NDFileHDF5Layout.h:44
name
Definition: makeDbAndEdl.py:232
list value
Definition: makeDbAndEdl.py:234
Definition: Codec.h:20