h5typer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- h5typer/__init__.py +49 -0
- h5typer/io.py +397 -0
- h5typer-0.1.0.dist-info/METADATA +319 -0
- h5typer-0.1.0.dist-info/RECORD +7 -0
- h5typer-0.1.0.dist-info/WHEEL +5 -0
- h5typer-0.1.0.dist-info/licenses/LICENSE +21 -0
- h5typer-0.1.0.dist-info/top_level.txt +1 -0
h5typer/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
h5typer: Automatic Type Mapping for Python Objects to HDF5
|
|
3
|
+
|
|
4
|
+
A lightweight, robust package for saving and loading nested Python dictionaries
|
|
5
|
+
to/from HDF5 files. Automatically handles type conversion for numpy arrays,
|
|
6
|
+
pandas DataFrames/Series, and standard Python types.
|
|
7
|
+
|
|
8
|
+
Features:
|
|
9
|
+
- Save and load arbitrarily nested dictionaries
|
|
10
|
+
- Automatic type conversion (numpy, pandas, strings, None, etc.)
|
|
11
|
+
- Support for complex nested structures
|
|
12
|
+
- Robust handling of various data types
|
|
13
|
+
|
|
14
|
+
Basic Usage
|
|
15
|
+
-----------
|
|
16
|
+
>>> import h5typer
|
|
17
|
+
>>>
|
|
18
|
+
>>> # Save data
|
|
19
|
+
>>> data = {'experiment': {'results': np.array([1, 2, 3])}}
|
|
20
|
+
>>> h5typer.save_data('output.h5', data)
|
|
21
|
+
>>>
|
|
22
|
+
>>> # Load data
|
|
23
|
+
>>> loaded_data = h5typer.load_data('output.h5')
|
|
24
|
+
|
|
25
|
+
Object-Oriented Usage
|
|
26
|
+
--------------------
|
|
27
|
+
>>> from h5typer import H5Typer
|
|
28
|
+
>>>
|
|
29
|
+
>>> io_handler = H5Typer()
|
|
30
|
+
>>> io_handler.save_data('output.h5', data)
|
|
31
|
+
>>> loaded = io_handler.load_data('output.h5')
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
__version__ = "0.1.0"
|
|
35
|
+
__author__ = "HiC-SCA Project"
|
|
36
|
+
__license__ = "MIT"
|
|
37
|
+
|
|
38
|
+
from .io import (
|
|
39
|
+
H5Typer,
|
|
40
|
+
save_data,
|
|
41
|
+
load_data
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
'H5Typer',
|
|
46
|
+
'save_data',
|
|
47
|
+
'load_data',
|
|
48
|
+
'__version__'
|
|
49
|
+
]
|
h5typer/io.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""
|
|
2
|
+
h5typer I/O Module
|
|
3
|
+
|
|
4
|
+
Handles saving and loading nested Python dictionaries to/from HDF5 files.
|
|
5
|
+
Provides automatic type mapping between Python objects and HDF5 data types.
|
|
6
|
+
Compatible with numpy arrays, pandas DataFrames/Series, and standard Python types.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import pandas as pd
|
|
11
|
+
import h5py
|
|
12
|
+
import types
|
|
13
|
+
from typing import Dict, Optional, Any
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class H5Typer:
|
|
18
|
+
"""
|
|
19
|
+
Handles saving and loading of nested Python dictionaries to HDF5.
|
|
20
|
+
|
|
21
|
+
Provides automatic type conversion and mapping between Python objects
|
|
22
|
+
and HDF5 data types.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self):
|
|
26
|
+
"""Initialize the data I/O handler."""
|
|
27
|
+
self.version = 1.0
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def _convert_npstr_to_byte(str_array: np.ndarray) -> np.ndarray:
|
|
31
|
+
"""Convert numpy string array to bytes for HDF5 compatibility."""
|
|
32
|
+
return np.char.encode(str_array, encoding='utf-8')
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def _convert_npbyte_to_str(byte_array: np.ndarray) -> np.ndarray:
|
|
36
|
+
"""Convert numpy bytes array to string."""
|
|
37
|
+
return np.char.decode(byte_array, encoding='utf-8')
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def _convert_obj_to_dtypes(obj_array: np.ndarray) -> np.ndarray:
|
|
41
|
+
"""Convert object dtype arrays to appropriate types."""
|
|
42
|
+
current_array = np.array(obj_array.tolist())
|
|
43
|
+
current_type = type(current_array.dtype)
|
|
44
|
+
|
|
45
|
+
type_handlers = {
|
|
46
|
+
str: lambda x: x.encode('utf-8'),
|
|
47
|
+
np.dtypes.StrDType: H5Typer._convert_npstr_to_byte,
|
|
48
|
+
types.NoneType: lambda x: h5py.Empty('f'),
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if current_type == np.dtypes.ObjectDType:
|
|
52
|
+
return obj_array
|
|
53
|
+
elif current_type in type_handlers:
|
|
54
|
+
return type_handlers[current_type](current_array)
|
|
55
|
+
else:
|
|
56
|
+
return current_array
|
|
57
|
+
|
|
58
|
+
def _convert_to_h5_compatible(self, obj: Any) -> tuple:
|
|
59
|
+
"""
|
|
60
|
+
Convert Python objects to HDF5-compatible format.
|
|
61
|
+
|
|
62
|
+
Returns
|
|
63
|
+
-------
|
|
64
|
+
tuple
|
|
65
|
+
(data_type, converted_obj)
|
|
66
|
+
"""
|
|
67
|
+
data_type = ''
|
|
68
|
+
converted_obj = obj
|
|
69
|
+
|
|
70
|
+
if isinstance(obj, np.ndarray):
|
|
71
|
+
data_type = 'numpy'
|
|
72
|
+
dtype_type = type(obj.dtype)
|
|
73
|
+
|
|
74
|
+
if dtype_type == np.dtypes.StrDType:
|
|
75
|
+
converted_obj = self._convert_npstr_to_byte(obj)
|
|
76
|
+
elif dtype_type == np.dtypes.ObjectDType:
|
|
77
|
+
converted_obj = self._convert_obj_to_dtypes(obj)
|
|
78
|
+
|
|
79
|
+
elif isinstance(obj, dict):
|
|
80
|
+
data_type = 'dict'
|
|
81
|
+
|
|
82
|
+
elif isinstance(obj, pd.core.generic.NDFrame):
|
|
83
|
+
values = obj.values
|
|
84
|
+
index = obj.index.values
|
|
85
|
+
name = obj.index.name
|
|
86
|
+
|
|
87
|
+
if isinstance(obj, pd.DataFrame):
|
|
88
|
+
data_type = 'pandas dataframe'
|
|
89
|
+
columns = obj.columns.values
|
|
90
|
+
converted_obj = {'values': values, 'index': index, 'name': name, 'columns': columns}
|
|
91
|
+
|
|
92
|
+
elif isinstance(obj, pd.Series):
|
|
93
|
+
data_type = 'pandas series'
|
|
94
|
+
converted_obj = {'values': values, 'index': index, 'name': name}
|
|
95
|
+
|
|
96
|
+
elif isinstance(obj, types.NoneType):
|
|
97
|
+
data_type = 'None'
|
|
98
|
+
converted_obj = h5py.Empty('f')
|
|
99
|
+
|
|
100
|
+
elif isinstance(obj, str):
|
|
101
|
+
data_type = 'str'
|
|
102
|
+
converted_obj = obj.encode('utf-8')
|
|
103
|
+
|
|
104
|
+
elif isinstance(obj, (int, float, np.integer, np.floating)):
|
|
105
|
+
data_type = 'scalar'
|
|
106
|
+
converted_obj = np.array(obj)
|
|
107
|
+
|
|
108
|
+
elif isinstance(obj, list) or isinstance(obj, tuple):
|
|
109
|
+
obj = np.array(obj)
|
|
110
|
+
data_type = 'object'
|
|
111
|
+
dtype_type = type(obj.dtype)
|
|
112
|
+
|
|
113
|
+
if dtype_type == np.dtypes.StrDType:
|
|
114
|
+
converted_obj = self._convert_npstr_to_byte(obj)
|
|
115
|
+
elif dtype_type == np.dtypes.ObjectDType:
|
|
116
|
+
converted_obj = self._convert_obj_to_dtypes(obj)
|
|
117
|
+
|
|
118
|
+
else:
|
|
119
|
+
data_type = 'object'
|
|
120
|
+
converted_obj = np.array(obj)
|
|
121
|
+
|
|
122
|
+
return data_type, converted_obj
|
|
123
|
+
|
|
124
|
+
def _flatten_dict(self, input_dict: Dict, parent_address: str = "") -> tuple:
|
|
125
|
+
"""
|
|
126
|
+
Flatten nested dictionary for HDF5 storage.
|
|
127
|
+
|
|
128
|
+
Parameters
|
|
129
|
+
----------
|
|
130
|
+
input_dict : dict
|
|
131
|
+
Nested dictionary to flatten
|
|
132
|
+
parent_address : str
|
|
133
|
+
Current hierarchical path
|
|
134
|
+
|
|
135
|
+
Returns
|
|
136
|
+
-------
|
|
137
|
+
tuple
|
|
138
|
+
(meta_dict, output_dict) containing metadata and flattened data
|
|
139
|
+
"""
|
|
140
|
+
output_dict = {}
|
|
141
|
+
meta_dict = {}
|
|
142
|
+
|
|
143
|
+
if len(input_dict) == 0:
|
|
144
|
+
meta_dict[parent_address] = 'dict'
|
|
145
|
+
else:
|
|
146
|
+
for key, value in input_dict.items():
|
|
147
|
+
data_type, current_value = self._convert_to_h5_compatible(value)
|
|
148
|
+
current_full_address = parent_address + '/' + str(key)
|
|
149
|
+
|
|
150
|
+
if isinstance(current_value, dict):
|
|
151
|
+
temp_meta_dict, temp_output_dict = self._flatten_dict(
|
|
152
|
+
current_value, current_full_address
|
|
153
|
+
)
|
|
154
|
+
output_dict.update(temp_output_dict)
|
|
155
|
+
|
|
156
|
+
# For pandas types, store metadata at current level
|
|
157
|
+
if 'pandas' in data_type:
|
|
158
|
+
meta_dict[current_full_address] = data_type
|
|
159
|
+
else:
|
|
160
|
+
meta_dict.update(temp_meta_dict)
|
|
161
|
+
else:
|
|
162
|
+
output_dict[current_full_address] = current_value
|
|
163
|
+
meta_dict[current_full_address] = data_type
|
|
164
|
+
|
|
165
|
+
if parent_address == "":
|
|
166
|
+
meta_dict = {'/__meta__' + key: val for key, val in meta_dict.items()}
|
|
167
|
+
|
|
168
|
+
return meta_dict, output_dict
|
|
169
|
+
|
|
170
|
+
def save_data(self, filename: str, data_dict: Dict, update: bool = False) -> None:
|
|
171
|
+
"""
|
|
172
|
+
Save nested dictionary to HDF5 file.
|
|
173
|
+
|
|
174
|
+
Parameters
|
|
175
|
+
----------
|
|
176
|
+
filename : str
|
|
177
|
+
Path to HDF5 file
|
|
178
|
+
data_dict : dict
|
|
179
|
+
Nested dictionary containing data to save
|
|
180
|
+
update : bool, optional
|
|
181
|
+
If True, update existing file; if False, overwrite (default: False)
|
|
182
|
+
"""
|
|
183
|
+
write_mode = 'a' if update else 'w'
|
|
184
|
+
|
|
185
|
+
meta_dict, flat_dict = self._flatten_dict(data_dict)
|
|
186
|
+
|
|
187
|
+
with h5py.File(filename, write_mode) as h5_file:
|
|
188
|
+
h5_file.update(flat_dict)
|
|
189
|
+
h5_file.update(meta_dict)
|
|
190
|
+
h5_file['__version__'] = self.version
|
|
191
|
+
|
|
192
|
+
def _auto_input_type_converter(self, obj: Any) -> Any:
|
|
193
|
+
"""Convert HDF5 objects back to Python types."""
|
|
194
|
+
obj_type = type(obj)
|
|
195
|
+
|
|
196
|
+
if obj_type == np.ndarray:
|
|
197
|
+
dtype_type = type(obj.dtype)
|
|
198
|
+
|
|
199
|
+
if dtype_type == np.dtypes.BytesDType:
|
|
200
|
+
return self._convert_npbyte_to_str(obj)
|
|
201
|
+
elif dtype_type == np.dtypes.StrDType:
|
|
202
|
+
return obj
|
|
203
|
+
|
|
204
|
+
elif obj_type == bytes:
|
|
205
|
+
return obj.decode('utf-8')
|
|
206
|
+
elif isinstance(obj, h5py._hl.base.Empty):
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
return obj
|
|
210
|
+
|
|
211
|
+
def _object_input_handler(self, data_type: str, var_name: str, data_dict: Dict) -> Any:
|
|
212
|
+
"""Handle object type reconstruction."""
|
|
213
|
+
data_obj = self._auto_input_type_converter(data_dict[var_name])
|
|
214
|
+
|
|
215
|
+
if (data_type == 'object') and (type(data_obj) == np.ndarray):
|
|
216
|
+
return data_obj.tolist()
|
|
217
|
+
|
|
218
|
+
elif (data_type in ('object', 'scalar')) and isinstance(data_obj, np.generic):
|
|
219
|
+
return data_obj.item()
|
|
220
|
+
|
|
221
|
+
return data_obj
|
|
222
|
+
|
|
223
|
+
def _str_input_handler(self, data_type: str, var_name: str, data_dict: Dict) -> str:
|
|
224
|
+
"""Handle string type reconstruction."""
|
|
225
|
+
return data_dict[var_name].decode('utf-8')
|
|
226
|
+
|
|
227
|
+
def _dict_input_handler(self, data_type: str, var_name: str, data_dict: Dict) -> dict:
|
|
228
|
+
"""Handle empty dict reconstruction."""
|
|
229
|
+
return {}
|
|
230
|
+
|
|
231
|
+
def _pandas_input_handler(self, data_type: str, var_name: str, data_dict: Dict):
|
|
232
|
+
"""Handle pandas DataFrame and Series reconstruction."""
|
|
233
|
+
values = data_dict[var_name + '/values']
|
|
234
|
+
name = self._auto_input_type_converter(data_dict[var_name + '/name'])
|
|
235
|
+
index = self._auto_input_type_converter(data_dict[var_name + '/index'])
|
|
236
|
+
|
|
237
|
+
if len(index.shape) > 1:
|
|
238
|
+
index = pd.MultiIndex.from_arrays(index.T, names=name)
|
|
239
|
+
else:
|
|
240
|
+
index = pd.Index(index, name=name)
|
|
241
|
+
|
|
242
|
+
if data_type == 'pandas series':
|
|
243
|
+
return pd.Series(values, index=index)
|
|
244
|
+
|
|
245
|
+
elif data_type == 'pandas dataframe':
|
|
246
|
+
columns = self._auto_input_type_converter(data_dict[var_name + '/columns'])
|
|
247
|
+
|
|
248
|
+
if len(columns.shape) > 1:
|
|
249
|
+
columns = pd.MultiIndex.from_arrays(columns.T)
|
|
250
|
+
|
|
251
|
+
return pd.DataFrame(values, index=index, columns=columns)
|
|
252
|
+
|
|
253
|
+
else:
|
|
254
|
+
raise ValueError(f"Invalid pandas data type: {data_type}")
|
|
255
|
+
|
|
256
|
+
def _build_dict_recursive(self, name_list: list, value: Any, existing_dict: Optional[Dict] = None) -> Dict:
|
|
257
|
+
"""Recursively rebuild nested dictionary structure."""
|
|
258
|
+
if existing_dict is None:
|
|
259
|
+
existing_dict = {}
|
|
260
|
+
|
|
261
|
+
current_key = name_list[0]
|
|
262
|
+
|
|
263
|
+
if len(name_list) > 1:
|
|
264
|
+
if current_key in existing_dict:
|
|
265
|
+
existing_dict[current_key].update(
|
|
266
|
+
self._build_dict_recursive(name_list[1:], value, existing_dict[current_key])
|
|
267
|
+
)
|
|
268
|
+
else:
|
|
269
|
+
existing_dict[current_key] = self._build_dict_recursive(name_list[1:], value)
|
|
270
|
+
else:
|
|
271
|
+
existing_dict[current_key] = value
|
|
272
|
+
|
|
273
|
+
return existing_dict
|
|
274
|
+
|
|
275
|
+
def _reload_from_flat_dict(self, meta_dict: Dict, data_dict: Dict) -> Dict:
|
|
276
|
+
"""Reconstruct nested dictionary from flattened HDF5 data."""
|
|
277
|
+
input_handlers = {
|
|
278
|
+
'numpy': self._object_input_handler,
|
|
279
|
+
'object': self._object_input_handler,
|
|
280
|
+
'dict': self._dict_input_handler,
|
|
281
|
+
'str': self._str_input_handler,
|
|
282
|
+
'None': self._object_input_handler,
|
|
283
|
+
'scalar': self._object_input_handler,
|
|
284
|
+
'pandas series': self._pandas_input_handler,
|
|
285
|
+
'pandas dataframe': self._pandas_input_handler,
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
processed_dict = {}
|
|
289
|
+
|
|
290
|
+
for var_full_name, data_type in meta_dict.items():
|
|
291
|
+
handler = input_handlers.get(data_type, self._object_input_handler)
|
|
292
|
+
var_name_list = var_full_name.split('/')
|
|
293
|
+
|
|
294
|
+
# Convert string keys that are integers back to int
|
|
295
|
+
for idx, var_name in enumerate(var_name_list):
|
|
296
|
+
if var_name.isdigit():
|
|
297
|
+
var_name_list[idx] = int(var_name)
|
|
298
|
+
|
|
299
|
+
var_name = var_name_list[0]
|
|
300
|
+
value = handler(data_type, var_full_name, data_dict)
|
|
301
|
+
|
|
302
|
+
if len(var_name_list) > 1:
|
|
303
|
+
if var_name in processed_dict:
|
|
304
|
+
current_value = self._build_dict_recursive(
|
|
305
|
+
var_name_list[1:], value, processed_dict[var_name]
|
|
306
|
+
)
|
|
307
|
+
else:
|
|
308
|
+
current_value = self._build_dict_recursive(var_name_list[1:], value)
|
|
309
|
+
else:
|
|
310
|
+
current_value = value
|
|
311
|
+
|
|
312
|
+
processed_dict[var_name] = current_value
|
|
313
|
+
|
|
314
|
+
return processed_dict
|
|
315
|
+
|
|
316
|
+
def load_data(self, filename: str) -> Dict:
|
|
317
|
+
"""
|
|
318
|
+
Load nested dictionary from HDF5 file.
|
|
319
|
+
|
|
320
|
+
Parameters
|
|
321
|
+
----------
|
|
322
|
+
filename : str
|
|
323
|
+
Path to HDF5 file
|
|
324
|
+
|
|
325
|
+
Returns
|
|
326
|
+
-------
|
|
327
|
+
dict
|
|
328
|
+
Nested dictionary containing loaded data
|
|
329
|
+
"""
|
|
330
|
+
with h5py.File(filename, 'r') as h5_file:
|
|
331
|
+
# Check version
|
|
332
|
+
if '__version__' in h5_file:
|
|
333
|
+
version = h5_file['__version__'][()]
|
|
334
|
+
if version != self.version:
|
|
335
|
+
print(f"Warning: File version {version} differs from current version {self.version}")
|
|
336
|
+
|
|
337
|
+
# Collect metadata and data
|
|
338
|
+
meta_dict = {}
|
|
339
|
+
data_dict = {}
|
|
340
|
+
|
|
341
|
+
def visitor(name, obj):
|
|
342
|
+
if isinstance(obj, h5py.Dataset):
|
|
343
|
+
address_list = name.split('/')
|
|
344
|
+
|
|
345
|
+
if address_list[0] == '__meta__':
|
|
346
|
+
meta_dict['/'.join(address_list[1:])] = obj[()].decode('utf-8')
|
|
347
|
+
elif name != '__version__':
|
|
348
|
+
data_dict[name] = obj[()]
|
|
349
|
+
|
|
350
|
+
h5_file.visititems(visitor)
|
|
351
|
+
|
|
352
|
+
return self._reload_from_flat_dict(meta_dict, data_dict)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
# Convenience functions for direct module usage
|
|
356
|
+
def save_data(filename: str, data_dict: Dict, update: bool = False) -> None:
|
|
357
|
+
"""
|
|
358
|
+
Save nested dictionary to HDF5 file (convenience function).
|
|
359
|
+
|
|
360
|
+
Parameters
|
|
361
|
+
----------
|
|
362
|
+
filename : str
|
|
363
|
+
Path to HDF5 file
|
|
364
|
+
data_dict : dict
|
|
365
|
+
Nested dictionary to save
|
|
366
|
+
update : bool, optional
|
|
367
|
+
Update existing file if True
|
|
368
|
+
|
|
369
|
+
Examples
|
|
370
|
+
--------
|
|
371
|
+
>>> data = {'results': {'exp1': np.array([1, 2, 3])}}
|
|
372
|
+
>>> save_data('output.h5', data)
|
|
373
|
+
"""
|
|
374
|
+
io_handler = H5Typer()
|
|
375
|
+
io_handler.save_data(filename, data_dict, update=update)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def load_data(filename: str) -> Dict:
|
|
379
|
+
"""
|
|
380
|
+
Load nested dictionary from HDF5 file (convenience function).
|
|
381
|
+
|
|
382
|
+
Parameters
|
|
383
|
+
----------
|
|
384
|
+
filename : str
|
|
385
|
+
Path to HDF5 file
|
|
386
|
+
|
|
387
|
+
Returns
|
|
388
|
+
-------
|
|
389
|
+
dict
|
|
390
|
+
Loaded nested dictionary
|
|
391
|
+
|
|
392
|
+
Examples
|
|
393
|
+
--------
|
|
394
|
+
>>> data = load_data('output.h5')
|
|
395
|
+
"""
|
|
396
|
+
io_handler = H5Typer()
|
|
397
|
+
return io_handler.load_data(filename)
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: h5typer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Automatic type mapping for Python objects to HDF5
|
|
5
|
+
Author-email: Justin Chan <chan.waisoon@qst.go.jp>, Hidetoshi Kono <kono.hidetoshi@qst.go.jp>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/iQLS-MMS/h5typer
|
|
8
|
+
Keywords: hdf5,type-mapping,numpy,pandas,serialization
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Scientific/Engineering
|
|
13
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy>=1.19.0
|
|
23
|
+
Requires-Dist: pandas>=1.0.0
|
|
24
|
+
Requires-Dist: h5py>=3.0.0
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# h5typer
|
|
28
|
+
|
|
29
|
+
**Automatic type mapping for Python objects to HDF5**
|
|
30
|
+
|
|
31
|
+
h5typer is a lightweight Python package that provides seamless type conversion when saving and loading nested Python dictionaries to/from HDF5 files. It automatically handles the complexity of mapping Python types (numpy arrays, pandas DataFrames/Series, None, etc.) to HDF5-compatible formats.
|
|
32
|
+
|
|
33
|
+
Originally developed as part of the [HiC-SCA](https://github.com/iQLS-MMS/hic-sca) package for Hi-C analysis, h5typer has been extracted as a standalone, reusable package that can be used in any Python project requiring HDF5 I/O with automatic type handling.
|
|
34
|
+
|
|
35
|
+
## Features
|
|
36
|
+
|
|
37
|
+
- **Automatic Type Conversion**: Transparently handles numpy arrays, pandas DataFrames/Series, strings, None, and standard Python types
|
|
38
|
+
- **Nested Dictionary Support**: Save and load arbitrarily nested dictionary structures
|
|
39
|
+
- **Robust**: Handles edge cases like empty arrays, None values, and object dtypes
|
|
40
|
+
- **Simple API**: Just two main functions - `save_data()` and `load_data()`
|
|
41
|
+
- **Efficient**: Uses HDF5's hierarchical storage for fast I/O
|
|
42
|
+
|
|
43
|
+
## Why h5typer?
|
|
44
|
+
|
|
45
|
+
Standard HDF5 libraries like h5py require manual handling of type conversions:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
# Without h5typer - manual type handling
|
|
49
|
+
import h5py
|
|
50
|
+
|
|
51
|
+
with h5py.File('data.h5', 'w') as f:
|
|
52
|
+
f.create_dataset('array', data=np.array([1, 2, 3]))
|
|
53
|
+
f.create_dataset('string', data='hello'.encode('utf-8')) # Must encode
|
|
54
|
+
# Complex types like pandas DataFrames require custom serialization
|
|
55
|
+
|
|
56
|
+
# With h5typer - automatic!
|
|
57
|
+
import h5typer
|
|
58
|
+
|
|
59
|
+
data = {
|
|
60
|
+
'array': np.array([1, 2, 3]),
|
|
61
|
+
'string': 'hello',
|
|
62
|
+
'dataframe': pd.DataFrame({'A': [1, 2]})
|
|
63
|
+
}
|
|
64
|
+
h5typer.save_data('data.h5', data)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
h5typer handles all the boilerplate, letting you focus on your data.
|
|
68
|
+
|
|
69
|
+
## Installation
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# Install from source
|
|
73
|
+
git clone https://github.com/iQLS-MMS/h5typer.git
|
|
74
|
+
cd h5typer
|
|
75
|
+
pip install .
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Requirements
|
|
79
|
+
|
|
80
|
+
- Python >= 3.7
|
|
81
|
+
- numpy >= 1.19.0
|
|
82
|
+
- pandas >= 1.0.0
|
|
83
|
+
- h5py >= 3.0.0
|
|
84
|
+
|
|
85
|
+
## Quick Start
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
import h5typer
|
|
89
|
+
import numpy as np
|
|
90
|
+
import pandas as pd
|
|
91
|
+
|
|
92
|
+
# Create some data
|
|
93
|
+
data = {
|
|
94
|
+
'experiment': {
|
|
95
|
+
'results': np.array([1, 2, 3, 4, 5]),
|
|
96
|
+
'metadata': {
|
|
97
|
+
'name': 'Test Experiment',
|
|
98
|
+
'date': '2025-10-27',
|
|
99
|
+
'valid': True
|
|
100
|
+
},
|
|
101
|
+
'dataframe': pd.DataFrame({
|
|
102
|
+
'A': [1, 2, 3],
|
|
103
|
+
'B': [4, 5, 6]
|
|
104
|
+
}),
|
|
105
|
+
'empty_value': None
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
# Save to HDF5
|
|
110
|
+
h5typer.save_data('output.h5', data)
|
|
111
|
+
|
|
112
|
+
# Load from HDF5
|
|
113
|
+
loaded_data = h5typer.load_data('output.h5')
|
|
114
|
+
|
|
115
|
+
# Access your data - it's exactly as you saved it!
|
|
116
|
+
print(loaded_data['experiment']['results']) # array([1, 2, 3, 4, 5])
|
|
117
|
+
print(loaded_data['experiment']['metadata']['name']) # 'Test Experiment'
|
|
118
|
+
print(loaded_data['experiment']['dataframe']) # Original DataFrame
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## API Reference
|
|
122
|
+
|
|
123
|
+
### Functions
|
|
124
|
+
|
|
125
|
+
#### `save_data(filename, data_dict, update=False)`
|
|
126
|
+
|
|
127
|
+
Save a nested dictionary to an HDF5 file.
|
|
128
|
+
|
|
129
|
+
**Parameters:**
|
|
130
|
+
- `filename` (str): Path to the HDF5 file
|
|
131
|
+
- `data_dict` (dict): Nested dictionary to save
|
|
132
|
+
- `update` (bool, optional): If True, update existing file; if False, overwrite. Default: False
|
|
133
|
+
|
|
134
|
+
**Example:**
|
|
135
|
+
```python
|
|
136
|
+
import h5typer
|
|
137
|
+
|
|
138
|
+
data = {'key': 'value', 'array': np.array([1, 2, 3])}
|
|
139
|
+
h5typer.save_data('mydata.h5', data)
|
|
140
|
+
|
|
141
|
+
# Update existing file
|
|
142
|
+
more_data = {'new_key': 'new_value'}
|
|
143
|
+
h5typer.save_data('mydata.h5', more_data, update=True)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
#### `load_data(filename)`
|
|
147
|
+
|
|
148
|
+
Load a nested dictionary from an HDF5 file.
|
|
149
|
+
|
|
150
|
+
**Parameters:**
|
|
151
|
+
- `filename` (str): Path to the HDF5 file
|
|
152
|
+
|
|
153
|
+
**Returns:**
|
|
154
|
+
- `dict`: The loaded nested dictionary
|
|
155
|
+
|
|
156
|
+
**Example:**
|
|
157
|
+
```python
|
|
158
|
+
import h5typer
|
|
159
|
+
|
|
160
|
+
data = h5typer.load_data('mydata.h5')
|
|
161
|
+
print(data['key']) # 'value'
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Class API
|
|
165
|
+
|
|
166
|
+
For more control, use the `H5Typer` class directly:
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from h5typer import H5Typer
|
|
170
|
+
|
|
171
|
+
# Create instance
|
|
172
|
+
io_handler = H5Typer()
|
|
173
|
+
|
|
174
|
+
# Save data
|
|
175
|
+
io_handler.save_data('output.h5', my_dict)
|
|
176
|
+
|
|
177
|
+
# Load data
|
|
178
|
+
loaded = io_handler.load_data('output.h5')
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## Supported Types
|
|
182
|
+
|
|
183
|
+
h5typer automatically handles the following Python types:
|
|
184
|
+
|
|
185
|
+
| Python Type | HDF5 Storage | Notes |
|
|
186
|
+
|-------------|--------------|-------|
|
|
187
|
+
| `numpy.ndarray` | Dataset | All dtypes supported, including object arrays |
|
|
188
|
+
| `pandas.DataFrame` | Group with datasets | Index, columns, and values preserved |
|
|
189
|
+
| `pandas.Series` | Group with datasets | Index and values preserved |
|
|
190
|
+
| `dict` | Group | Nested dictionaries become HDF5 groups |
|
|
191
|
+
| `str` | Dataset (bytes) | UTF-8 encoded |
|
|
192
|
+
| `int`, `float` | Dataset | Stored as numpy scalars |
|
|
193
|
+
| `None` | h5py.Empty | Preserved on load |
|
|
194
|
+
| `list`, `tuple` | Dataset | Converted to numpy arrays, all elements must be of the same type |
|
|
195
|
+
|
|
196
|
+
## Type Conversion Details
|
|
197
|
+
|
|
198
|
+
### NumPy Arrays
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
# String arrays
|
|
202
|
+
str_array = np.array(['a', 'b', 'c'])
|
|
203
|
+
# Automatically converted to bytes for HDF5
|
|
204
|
+
|
|
205
|
+
# Float64 arrays
|
|
206
|
+
float_array = np.array([1.0, 5.0, 3.0], dtype=np.float64)
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Pandas Objects
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
# DataFrames - index, columns, and values all preserved
|
|
213
|
+
df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
|
|
214
|
+
|
|
215
|
+
# Series - index and values preserved
|
|
216
|
+
series = pd.Series([1, 2, 3], index=['a', 'b', 'c'])
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### None Values
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
data = {
|
|
223
|
+
'value': None,
|
|
224
|
+
}
|
|
225
|
+
# None values are preserved through save/load
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
## Advanced Usage
|
|
229
|
+
|
|
230
|
+
### Nested Structures
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
data = {
|
|
234
|
+
'level1': {
|
|
235
|
+
'level2': {
|
|
236
|
+
'level3': {
|
|
237
|
+
'deep_array': np.array([1, 2, 3])
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
h5typer.save_data('nested.h5', data)
|
|
244
|
+
loaded = h5typer.load_data('nested.h5')
|
|
245
|
+
# Structure is preserved
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
### Updating Files
|
|
249
|
+
|
|
250
|
+
```python
|
|
251
|
+
# Initial save
|
|
252
|
+
h5typer.save_data('data.h5', {'key1': 'value1'})
|
|
253
|
+
|
|
254
|
+
# Add more data
|
|
255
|
+
h5typer.save_data('data.h5', {'key2': 'value2'}, update=True)
|
|
256
|
+
|
|
257
|
+
# Load gets both keys
|
|
258
|
+
data = h5typer.load_data('data.h5')
|
|
259
|
+
# {'key1': 'value1', 'key2': 'value2'}
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
### Integer Keys
|
|
263
|
+
|
|
264
|
+
```python
|
|
265
|
+
# Integer keys are preserved
|
|
266
|
+
data = {
|
|
267
|
+
100: 'hundred',
|
|
268
|
+
200: 'two hundred',
|
|
269
|
+
'string_key': 'value'
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
h5typer.save_data('intkeys.h5', data)
|
|
273
|
+
loaded = h5typer.load_data('intkeys.h5')
|
|
274
|
+
print(loaded[100]) # 'hundred' - key is still an integer
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
## Integration with HiC-SCA
|
|
278
|
+
|
|
279
|
+
h5typer is used by [HiC-SCA](https://github.com/iQLS-MMS/hic-sca) for all HDF5 I/O operations:
|
|
280
|
+
|
|
281
|
+
```python
|
|
282
|
+
# In HiC-SCA
|
|
283
|
+
from hicsca import HiCSCA
|
|
284
|
+
|
|
285
|
+
# Process Hi-C data
|
|
286
|
+
hicsca = HiCSCA("sample.hic", resolutions=[100000])
|
|
287
|
+
hicsca.process_all_chromosomes()
|
|
288
|
+
|
|
289
|
+
# Save results (uses h5typer internally)
|
|
290
|
+
hicsca.to_hdf5("results.h5")
|
|
291
|
+
|
|
292
|
+
# Load results (uses h5typer internally)
|
|
293
|
+
hicsca_loaded = HiCSCA.from_hdf5("results.h5")
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
The integration is transparent - HiC-SCA uses h5typer for automatic type mapping of complex nested dictionaries containing numpy arrays, pandas DataFrames, and metadata.
|
|
297
|
+
|
|
298
|
+
## License
|
|
299
|
+
|
|
300
|
+
MIT License
|
|
301
|
+
|
|
302
|
+
## Contributing
|
|
303
|
+
|
|
304
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
305
|
+
|
|
306
|
+
## Changelog
|
|
307
|
+
|
|
308
|
+
### Version 0.1.0
|
|
309
|
+
- Initial release
|
|
310
|
+
- Support for numpy arrays, pandas DataFrames/Series, None values
|
|
311
|
+
- Nested dictionary support
|
|
312
|
+
- Integer key preservation
|
|
313
|
+
- Update functionality
|
|
314
|
+
|
|
315
|
+
## See Also
|
|
316
|
+
|
|
317
|
+
- **HiC-SCA**: Hi-C Spectral Compartment Analysis package using h5typer
|
|
318
|
+
- **h5py**: Low-level HDF5 interface for Python
|
|
319
|
+
- **pandas**: Data analysis library with HDF5 support
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
h5typer/__init__.py,sha256=tG9uZADIDAW5c9Ph7b-8KpIvYyZ1BMA-Ycqjs51xCew,1132
|
|
2
|
+
h5typer/io.py,sha256=GDE72JreMxnAk8W2e7Iij4WyDggVQ95f3XtY1fRRv-E,13359
|
|
3
|
+
h5typer-0.1.0.dist-info/licenses/LICENSE,sha256=58XdpW1w9bAYTrokz315fY90xhh-SMsFh55EUiZL0V0,1106
|
|
4
|
+
h5typer-0.1.0.dist-info/METADATA,sha256=rzSzmQVLz0kA7vRkP0qKUHxyOID7qaiAizlr68V0ckc,8222
|
|
5
|
+
h5typer-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
6
|
+
h5typer-0.1.0.dist-info/top_level.txt,sha256=k4zAyVsea7DFRK3X9NEbLwF-L1GHBHurAV7-ZbfQQhE,8
|
|
7
|
+
h5typer-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Molecular Modelling and Simulation Team, iQLS, QST
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
h5typer
|