High-Level Functions for HDF5 Files

Prepared 11 July 2011 by Scot Martin for Windows 7, 64-bit, Mathematica version 8.0.1.

Email: scot_martin@harvard.edu.

Package

BeginPackage Context[...]

BeginPackage["HDF5HighLevel`",{"HDF5`","HDF5Extend`","NETLink`"}]

$HDF5HighLevelPackageVersion="V1.00"

(* loaded to allow member names to be extracted as Strings from IntPtr return of H5TgetMemberName *)
LoadNETType["System.Runtime.InteropServices.Marshal"]

Initialization

This functionality is used in CompoundDataType`Transform

LoadNETType["System.BitConverter"]

Usage Statements

DataSetDimensions::usage=
"DataSetDimensions[\"FileName\", \"PathToDataSet\"] returns the dimensions of the data set as a list."

DataSetDataTypeClass::usage=
"DataSetDataTypeClass[\"FileName\", \"PathToDataSet\"] returns a ClassCode. The legend for the ClassCode is as follows: {0, \"INTEGER\"}, {1, \"FLOAT\"}, {2, \"TIME\"}, {3, \"STRING\"}, {4, \"BITFIELD\"}, {5, \"OPAQUE\"}, {6, \"COMPOUND\"}, {7, \"REFERENCE\"}, {8, \"ENUM\"}, {9, \"VLEN\"}, {10, \"ARRAY\"}."

ReadHyperSlab::usage=
"ReadHyperSlab[\"FileName\", \"PathToDataSet\", \"OffsetList\", \"CountList\"] returns a hyperslab of data from the specified dataset.\n\nThe offset index runs from 1 forward, which is standard Mathematica syntax, rather than 0 forward as in the HDFView program. In case of error, the returned dataset is full of 0's.\n\nThis function is set up to work on datasets of integer32, integer64, single floating point, and double floating point. Extension to integer or other primitive data types would be easy to implement.\n\nThis function takes the option \"FinalIndexFull\", which is False by default. In the case of True, then OffsetList and CountList should be reduced by one in length and the function will provide the correct values such that the full data range is read in this dimension."

ReadColumnRank1::usage=
"ReadColumnRank1[\"FileName\", \"PathToDataSet\"] returns the data from the specified dataset of rank 1. This function uses ReadHyperSlab[...]."

ReadColumnRank2::usage=
"ReadColumnRank2[\"FileName\", \"PathToDataSet\", \"Column\"] returns one column of data from the specified dataset of rank 2. This function uses ReadHyperSlab[...]."

CompoundDataType`Information::usage=
"CompoundDataType`Information[\"FileName\", \"PathToDataSet\"] returns a list of information about the compound data types. The class codes are the same as for DataSetDateTypeClass. This function also indicates whether Strings are fixed (3) or variable length (-3) in its output of \"MemberDataTypeClass\"."

CompoundDataType`RawData::usage=
"CompoundDataType`RawData[\"FileName\", \"PathToDataSet\"] returns a list of rules as \"Data\" for the byte data (see following) as well as rules for \"MemberDataTypeClass\" and \"MemberMemorySize\" that are useful for transformation of byte data to values. The byte data are given as a list of lists. The first list corresponds to the first column of compound data. The second list goes with the second column of compound data and so on. (The user typically will not call this function directly but instead will call CompoundDataType`Values.)"

CompoundDataType`Transform::usage=
"CompoundDataType`Transform[\"RawDataObject\"] takes as input the output of CompoundDataType`RawData[...]. This function then returns a list of byte data transformed as appropriate into Integer, Float, and String. At present time, Integer32, Integer64, SingleFloat, DoubleFloat, and String are supported. (The user typically will not call this function directly but instead will call CompoundDataType`Values.)"

CompoundDataType`Values::usage=
"CompoundDataType`Values[\"FileName\", \"PathToDataSet\"] consecutively calls CompoundDataType`RawData[...] and CompoundDataType`Transform[...]. See notes for those functions."

NamesAndTypesAtOneNode::usage=
"NamesAndTypesAtOneNode[\"GroupOrFileID\"] returns a list of {{\"Name\", Type}, ...} for all subnodes of the node indicated by the GroupOrFileID. Types include: {0, \"GROUP\"}, {1, \"DATASET\"}, {2, \"DATATYPE\"}. (The user typically will not call this function directly but instead will call NamesAndTypes.)"

NamesAndTypes::usage=
"NamesAndTypesAtOneNode[\"FileName\"] returns a list of {{\"Name\", Type}, ...} for all nodes in file. For Type information, see NamesAndTypesAtOneNode[...]."

ReadAttributes::usage=
"ReadAttributes[\"FileName\", \"PathToObject\"] returns a list of information and values about the object. Use \".\" for the root directory for the object."

Begin Private Context[...]

Begin["Private`"]

Functions

DataSetDimensions

DataSetDimensions[filename:_String,dataSetPath:_String]:=

Module[

{fileID,dataSetID,dataSpaceID,dataSpaceDimensions,statusFlag},

H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},

H5DExtendOpen[
{dataSetID=H5Dopen[fileID, dataSetPath, H5PuDEFAULT]},

CompoundExpression[
(* get ID's *)
dataSpaceID=H5DgetSpace[dataSetID],
dataSpaceDimensions=H5SExtendGetSimpleExtentDims[dataSpaceID],

(* non-zero StatusFlag indicates an error of some sort occurred*)
statusFlag=H5Sclose[dataSpaceID] ,

(* function's return expression *)
If[Positive[statusFlag],-1,dataSpaceDimensions]
] (* close CompoundExpression *)
] (* close H5DExtendOpen *)
] (* close H5FExtendOpen *)
] (* close Module *)

DataSetDataTypeClass

DataSetDataTypeClass[filename:_String,dataSetPath:_String]:=

Module[

{fileID,dataSetID,dataTypeID,dataTypeClass,statusFlag},

H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},

H5DExtendOpen[
{dataSetID=H5Dopen[fileID, dataSetPath, H5PuDEFAULT]},

CompoundExpression[
(* get ID's *)
dataTypeID=H5DgetType[dataSetID],

(* 0=INTEGER, 1=FLOAT, etc. *)
dataTypeClass=H5TgetClass[dataTypeID] ,

(* non-zero StatusFlag indicates an error of some sort occurred*)
statusFlag=H5Tclose[dataTypeID] ,

(* function's return expression *)
If[Positive[statusFlag],-1,dataTypeClass]
](* close CompoundExpression *)
] (* close H5DExtendOpen *)
] (* close H5FExtendOpen *)
] (* close Module *)

ReadColumnRank1

ReadColumnRank1[filename:_String,dataSetPath:_String]:=

ReadHyperSlab[filename,dataSetPath,{},{},"FinalIndexFull"->True]

ReadColumnRank2

ReadColumnRank2[filename:_String,dataSetPath:_String,column:_Integer]:=

Flatten[ReadHyperSlab[filename,dataSetPath,{column},{1},"FinalIndexFull"->True]]

ReadHyperSlab

Options[ReadHyperSlab]={"FinalIndexFull"->False}

ReadHyperSlab[filename:_String,dataSetPath:_String,offsetInstructions:_List,countInstructions:_List,opts:OptionsPattern[]]:=

Module[

{fileID,dataSetID,dataTypeID,dataSpaceID,dataTypeClass,nativeDataTypeID,dataTypeByteSize,memorySpaceID,data,offset,count,dataSpaceDimensionsObject,statusFlag,lastItemCount},

H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},

H5DExtendOpen[
{dataSetID=H5Dopen[fileID, dataSetPath, H5PuDEFAULT]},

CompoundExpression[
(* get ID's *)
dataTypeID=H5DgetType[dataSetID],
dataSpaceID=H5DgetSpace[dataSetID],

(* 0=INTEGER, 1=FLOAT, etc. *)
dataTypeClass=H5TgetClass[dataTypeID] ,
nativeDataTypeID=H5TgetNativeType[dataTypeID,H5TuDEFAULT] ,
(* 4 bytes for Single, 8 bytes for Double, ... *)
dataTypeByteSize=H5TgetSize[dataTypeID],

If[
OptionValue["FinalIndexFull"],
(* option: True *)
CompoundExpression[
(* figure out length of last item in data set*)
lastItemCount=Last@H5SExtendGetSimpleExtentDims[dataSpaceID],
(* append the length final item to the count list*)
Set[{offset,count},{Append[offsetInstructions,1],Append[countInstructions,lastItemCount]}]
],
(* option: False *)
Set[{offset,count},{offsetInstructions,countInstructions}]
] (* end If[...]*),

(* set the region of the file to read *)
H5SselectHyperSlab[dataSpaceID,H5SuSET,offset-1,Null,count ,Null],

(* set up a similar region of space in the computer's memory *)
memorySpaceID=H5ScreateUsimple[Length@count,count,Null],

(* use the correct read function based on the deduced data type *)
Which[
dataTypeClass==0&&dataTypeByteSize==4,
data=H5DExtendRead[dataSetID,nativeDataTypeID,memorySpaceID,dataSpaceID,H5PuDEFAULT,count,"DataType"->Integer32],

dataTypeClass==0&&dataTypeByteSize==8,
data=H5DExtendRead[dataSetID,nativeDataTypeID,memorySpaceID,dataSpaceID,H5PuDEFAULT,count,"DataType"->Integer64],

dataTypeClass==1&&dataTypeByteSize==4,
data=H5DExtendRead[dataSetID,nativeDataTypeID,memorySpaceID,dataSpaceID,H5PuDEFAULT,count,"DataType"->SingleFloat],

dataTypeClass==1&&dataTypeByteSize==8,
data=H5DExtendRead[dataSetID,nativeDataTypeID,memorySpaceID,dataSpaceID,H5PuDEFAULT,count,"DataType"->DoubleFloat]
] (* close Which *),

(* non-zero StatusFlag indicates an error of some sort occurred*)
statusFlag=BitOr@@Abs[{H5Sclose[dataSpaceID],H5Sclose[memorySpaceID],H5Tclose[dataTypeID],H5Tclose[nativeDataTypeID]}] ,

(* function's return expression *)
If[Positive[statusFlag],-1,data]
](* close CompoundExpression *)
] (* close H5DExtendOpen *)
] (* close H5FExtendOpen *)
] (* close Module *)

CompoundDataType`Internal

CompoundDataType`Internal[fileID:_Integer,dataSetID:_Integer]:=

Module[

{dataSpaceID,dataTypeID,nativeDataTypeID,numMembers,memberIndexList,memberDataTypeIDList,memberNativeDataTypeIDList,memberDataTypeClass,memberDataTypeClassExpanded,returnList,statusFlag},

CompoundExpression[
dataSpaceID=H5DgetSpace[dataSetID],

dataTypeID=H5DgetType[dataSetID],

nativeDataTypeID=H5TgetNativeType[dataTypeID,H5TuDEFAULT],

numMembers=H5TgetNMembers[dataTypeID],

memberIndexList=Range[0,H5TgetNMembers[dataTypeID]-1],

memberDataTypeIDList=H5TgetMemberType[dataTypeID,#]&/@memberIndexList,

memberNativeDataTypeIDList=H5TgetNativeType[#,H5TuDEFAULT]&/@memberDataTypeIDList,

(* use -3 in place of 3 for variable length strings *)
memberDataTypeClass=(H5TgetMemberClass[dataTypeID,#]&/@memberIndexList),
memberDataTypeClassExpanded=
Replace[
(* set up a list of { {class,varStringFlag}, ...} *)
Thread[{memberDataTypeClass,H5TisVariableStr/@memberDataTypeIDList}],
(* match class (=3) and variable length string (>0) and assign -3, as necessary*)
{class:_,varString:_}:>If[class==3&&Positive[varString],-3,class],
{1}
],

returnList=
{
"NumberOfMembers"->numMembers,
"DataSpaceDimensions"-> H5SExtendGetSimpleExtentDims[dataSpaceID],
"MemoryDataTypeSize"-> H5TgetSize[nativeDataTypeID],
"MemberDataTypeClass"->memberDataTypeClassExpanded,
"MemberOffset"->(H5TgetMemberOffset[dataTypeID,#]&/@memberIndexList),
"MemberName"->(Marshal`PtrToStringAnsi/@(H5TgetMemberName[dataTypeID,#]&/@memberIndexList) ),
"MemberSize"->(H5TgetSize[#]&/@memberDataTypeIDList),
"MemberMemorySize"->(H5TgetSize[#]&/@memberNativeDataTypeIDList)
},

H5Tclose/@memberDataTypeIDList,

H5Tclose/@memberNativeDataTypeIDList,

(* non-zero StatusFlag indicates an error of some sort occurred*)
statusFlag=BitOr@@Abs[{H5Sclose[dataSpaceID],H5Tclose[dataTypeID],H5Tclose[nativeDataTypeID]}],

(* function's return expression *)
If[Positive[statusFlag],-1,Sort@returnList]
](* close CompoundExpression *)
] (* close Module *)

CompoundDataType`Information

CompoundDataType`Information[filename:_String,dataSetPath:_String]:=
Module[
{fileID,dataSetID,dataSpaceID,dataTypeID,nativeDataTypeID,numMembers,memberIndexList,memberDataTypeIDList,memberNativeDataTypeIDList,returnList,statusFlag},
H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},
H5DExtendOpen[
{dataSetID=H5Dopen[fileID, dataSetPath, H5PuDEFAULT]},
CompoundDataType`Internal[fileID,dataSetID]
] (* close H5DExtendOpen *)
] (* close H5FExtendOpen *)
] (* close Module *)

CompoundDataType`RawData

CompoundDataType`RawData[filename:_String,dataSetPath:_String]:=

Module[

{fileID,dataSetID,information,dataTypeID,memoryDataTypeID,rawDataDimensions,byteData,partitionList,formattedData,statusFlag},

H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},

H5DExtendOpen[
{dataSetID=H5Dopen[fileID, dataSetPath, H5PuDEFAULT]},

CompoundExpression[

information=CompoundDataType`Internal[fileID,dataSetID],

(* figure out dimensions for bytes to read input *)
rawDataDimensions=
Flatten[
{
OptionValue[information,"DataSpaceDimensions"],
OptionValue[information,"MemoryDataTypeSize"]
}
],

(* get information needed for H5DExtendRead *)
dataTypeID=H5DgetType[dataSetID],
memoryDataTypeID=H5TgetNativeType[dataTypeID,H5TuDEFAULT],

(* get as byte list formatted to rawDataDimensions *)
byteData=H5DExtendRead[dataSetID,memoryDataTypeID,H5SuALL,H5SuALL,H5PuDEFAULT,rawDataDimensions,"DataType"->Byte],

H5Tclose[dataTypeID],
H5Tclose[memoryDataTypeID],

(* further partition list according to size of each compound element *)
partitionList=
With[{ends=Accumulate[OptionValue[information,"MemberMemorySize"]]},
With[{starts=Prepend[Most[ends+1],1]},
Thread[{starts,ends}]
]],

formattedData=
Transpose[
Table[
Take[byteData[[i]],#]&/@partitionList,
{i,Length@byteData}
]
],

(*return*)

{Rule["Data",formattedData],Sequence@@FilterRules[information,{"MemberDataTypeClass","MemberMemorySize"}]}

](* close CompoundExpression *)
] (* close H5DExtendOpen *)
] (* close H5FExtendOpen *)
] (* close Module *)

CompoundDataType`Transform

CompoundDataType`Transform[rawDataObject:_List]:=

With[
{
memberDataTypeClass=OptionValue[rawDataObject,"MemberDataTypeClass"],
memberMemorySize=OptionValue[rawDataObject,"MemberMemorySize"],
data=OptionValue[rawDataObject,"Data"]
},

Table[
compoundDataTypeConverter[memberDataTypeClass[[i]]][memberMemorySize[[i]]]/@data[[i]],
{i,Length@data}
](* close Table *)

] (* close With's *)

http://msdn.microsoft.com/en-us/library/system.bitconverter.aspx

compoundDataTypeConverter[0][4][arg:_List]:=BitConverter`ToInt32[arg,0]

compoundDataTypeConverter[0][8][arg:_List]:=BitConverter`ToInt64[arg,0]

compoundDataTypeConverter[1][4][arg:_List]:=BitConverter`ToSingle[arg,0]

compoundDataTypeConverter[1][8][arg:_List]:=BitConverter`ToDouble[arg,0]

(* assumes that strings are terminated by 0: http://www.hdfgroup.org/HDF5/doc/RM/H5T/H5Tset_strpad.htm *)
compoundDataTypeConverter[3][_][arg:_List]:=FromCharacterCode[TakeWhile[arg,Positive]]

(* implementation should require IntPtr set up *)
compoundDataTypeConverter[-3][_][arg:_List]:=
"VariableLengthString is not supported at present, though certainly could be programmed."

CompoundDataType`Values

CompoundDataType`Values[filename:_String,dataSetPath:_String]:=

NETBlock[

With[
{
rawDataObject=CompoundDataType`RawData[filename,dataSetPath]
},

CompoundDataType`Transform[rawDataObject]

] (* close With *)

]

NamesAndTypesAtOneNode

NamesAndTypesAtOneNode[GroupOrFileID:_Integer]:=

NETBlock[

With[
{indexList=Range[0,H5GExtendGetNumObjs[GroupOrFileID]-1]},

Thread[
{
H5GExtendGetObjNameByIndex[GroupOrFileID,#]&/@indexList,
H5GgetObjTypeByIndex[GroupOrFileID,#]&/@indexList
}
]

] (* close With *)

](* close NETBlock *)

NamesAndTypes

NamesAndTypes[filename:_String]:=

NETBlock[

Module[

{fileID,returnList={},groupNamesThisLevel={"."},groupIDsThisLevel,itemsThisLevel={}},

H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},

CompoundExpression[

While[

(* so long as there are more groups, continue... *)
Length[groupNamesThisLevel]>0,

CompoundExpression[

(* get list of all group IDs for this level *)
groupIDsThisLevel=H5Gopen[fileID,#,H5PuDEFAULT]&/@groupNamesThisLevel,

(* code below has the effect of opening the group IDs on this level, reading in {Name,Type} and updating to {Group/Name,Type}*)
itemsThisLevel=
Flatten[
Table[
With[
{
(* function to add "group/" at front of name *)
groupNamePrepend=StringJoin[groupNamesThisLevel[[i]],"/",# (* item name *) ]&,
(* list of {name,type} at this ONE group node *)
namesAndTypesThisNode=NamesAndTypesAtOneNode[groupIDsThisLevel[[i]]]
},
(* update to "group/name" for ALL items at this ONE group node *)
MapAt[groupNamePrepend,# (* {name,type} *),1]&/@namesAndTypesThisNode
],
(* do updating across ALL group nodes at this ONE level *)
{i,Length@groupIDsThisLevel}
],
1
],

H5Gclose[#]&/@groupIDsThisLevel,

returnList=returnList~Join~itemsThisLevel,

(* set up for next loop through While[...] *)
groupNamesThisLevel=Cases[itemsThisLevel,{groupName:_,0}:>groupName]

] (* close CompoundExpression *) 
] (*close While*),

returnList

] (* close CompoundExpression *) 
] (* close H5FExtendOpen *)
] (* close Module *)
] (* close NETBlock *)

ReadAttributes

ReadAttributes[filename:_String,object:_String]:=

NETBlock[

Module[

{fileID,objectID,numAttributes,attributeIndexList,attributeIDlist,returnList,attributeDataTypeIDlist,attributeNativeDataTypeIDlist,attributeMemorySize,attributeDataSpaceIDlist,attributeDataSpaceDimensionsList,attributeDataTypeClassList,attributeDataTypeClassExpandedList,attributeValues},

H5FExtendOpen[
{fileID=H5Fopen[filename,H5FuACCuRDONLY,H5PuDEFAULT]},

CompoundExpression[
objectID=H5Oopen[fileID,object,H5PuDEFAULT],

numAttributes=H5AgetNumAttrs[objectID],
attributeIndexList=Range[0,numAttributes-1],

attributeIDlist=H5AopenByIndex[objectID,".",H5uNAME,H5uINCREASING,#,H5PuDEFAULT,H5PuDEFAULT]&/@attributeIndexList,

attributeDataTypeIDlist=H5AgetType[#]&/@attributeIDlist,

attributeNativeDataTypeIDlist=H5TgetNativeType[#,H5TuDEFAULT]&/@attributeDataTypeIDlist,
attributeMemorySize=(H5TgetSize[#]&/@attributeNativeDataTypeIDlist),

attributeDataSpaceIDlist=H5AgetSpace[#]&/@attributeIDlist,

attributeDataSpaceDimensionsList=(H5SExtendGetSimpleExtentDims[#]&/@attributeDataSpaceIDlist),

(* use -3 in place of 3 for variable length strings *)
attributeDataTypeClassList=(H5TgetClass[#]&/@attributeDataTypeIDlist),
attributeDataTypeClassExpandedList=
Replace[
(* set up a list of { {class,varStringFlag}, ...} *)
Thread[{attributeDataTypeClassList,H5TisVariableStr/@attributeDataTypeIDlist}],
(* match class (=3) and variable length string (>0) and assign -3, as necessary*)
{class:_,varString:_}:>If[class==3&&Positive[varString],-3,class],
{1}
],

attributeValues=
Thread[ReadAttributes`Internal`OneElement[attributeIDlist,attributeNativeDataTypeIDlist,attributeDataSpaceDimensionsList,attributeDataTypeClassExpandedList,attributeMemorySize]],

returnList=
{
"NumberOfAttributes"->numAttributes,
"AttributeDataSpaceDimensions"->attributeDataSpaceDimensionsList,
"AttributeDataTypeClass"->attributeDataTypeClassExpandedList,
"AttributeName"->(H5AExtendGetName[#]&/@attributeIDlist),
"AttributeSize"->(H5AgetStorageSize[#]&/@attributeIDlist),
"AttributeMemorySize"->attributeMemorySize,
"AttributeValues"->attributeValues
},

H5Aclose/@attributeIDlist,
H5Tclose/@attributeDataTypeIDlist,
H5Tclose/@attributeNativeDataTypeIDlist,
H5Sclose/@attributeDataSpaceIDlist,

H5Oclose[objectID],

returnList

](* close CompoundExpression *)
] (* close H5FExtendOpen *)
] (* close Module *)
] (* close NETBlock *)

ReadAttributes`Internal`OneElement

ReadAttributes`Internal`OneElement[attributeID:_Integer,memoryTypeID:_Integer,dimensions:_List,dataTypeClass:_Integer (*AttributeDataTypeClass*),dataTypeByteSize:_Integer (*AttributeMemorySize*)]:=

NETBlock[

Module[

{data},

CompoundExpression[

data=

(* use the correct read function based on the deduced data type *)
Which[
dataTypeClass==0&&dataTypeByteSize==4,
H5AExtendRead[attributeID,memoryTypeID,dimensions,"DataType"->Integer32],

dataTypeClass==0&&dataTypeByteSize==8,
H5AExtendRead[attributeID,memoryTypeID,dimensions,"DataType"->Integer64],

dataTypeClass==1&&dataTypeByteSize==4,
H5AExtendRead[attributeID,memoryTypeID,dimensions,"DataType"->SingleFloat],

dataTypeClass==1&&dataTypeByteSize==8,
H5AExtendRead[attributeID,memoryTypeID,dimensions,"DataType"->DoubleFloat],

dataTypeClass==3 (*fixed length string*),
FromCharacterCode[H5AExtendRead[attributeID,memoryTypeID,{dataTypeByteSize},"DataType"->Byte]],

True,
"UnsupportedDataType"
] (* close Which *),

If[Dimensions[data]=={1},First@data,data]

] (* close CompoundExpression *)

] (* close Module *)
] (* close NETBlock *)

End Private Context[...]

End[]

SetAttributes[
{DataSetDimensions,DataSetDataTypeClass,ReadColumnRank1,ReadColumnRank2,ReadHyperSlab,CompoundDataType`Information,CompoundDataType`RawData,CompoundDataType`Transform,CompoundDataType`Values,NamesAndTypesAtOneNode,NamesAndTypes,ReadAttributes,ReadAttributes`Internal`OneElement},
ReadProtected
]

EndPackage Context[...]

EndPackage[]