daio 0.0.2__tar.gz → 0.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: daio
3
- Version: 0.0.2
3
+ Version: 0.0.4
4
4
  Summary: video and data IO tools for Python
5
5
  Author: jlab.berlin
6
6
  License: MIT
@@ -10,7 +10,6 @@ Requires-Python: >=3.8
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
12
  Requires-Dist: numpy
13
- Requires-Dist: tqdm
14
13
  Requires-Dist: av
15
14
  Requires-Dist: h5py
16
15
  Requires-Dist: hdf5plugin
@@ -49,7 +48,7 @@ for i in range(20):
49
48
  writer.close()
50
49
  ```
51
50
 
52
- Read video:
51
+ Read video using speed-optimized array-like indexing or iteration:
53
52
  ```python
54
53
  reader = VideoReader('/path/to/video.mp4')
55
54
  frame_7 = reader[7]
@@ -72,22 +71,48 @@ with VideoReader('/path/to/video.mp4') as reader:
72
71
 
73
72
  ### HDF5 file IO
74
73
 
75
- Write a Python dict to a HDF5 file:
74
+ Lazily load HDF5 with a dict-like interface (contents are only loaded when accessed):
76
75
  ```python
77
- from daio.h5 import save_to_h5, load_from_h5, lazyh5
78
- some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
79
- save_to_h5('/path/to/datafile.h5', some_dict)
76
+ from daio.h5 import lazyh5
77
+ h5 = lazyh5('/path/to/datafile.h5')
78
+ b_loaded = h5['b']
79
+ e_loaded = h5['c']['e']
80
+ h5.keys()
80
81
  ```
81
82
 
82
- Load HDF5 contents to dict:
83
+ Create a new HDF5 file (or add items to existing file by setting argument `readonly=False`):
83
84
  ```python
84
- dict_loaded = load_from_h5('/path/to/datafile.h5', d)
85
+ h5 = lazyh5('test.h5')
86
+ h5['a'] = 1
87
+ h5['b'] = 'hello'
88
+ h5['c'] = {} # create subgroup
89
+ h5['c']['e'] = [2,3,4]
85
90
  ```
86
91
 
87
- Lazily load HDF5 with a dict-like interface (contents are only loaded when accessed):
92
+ Load entire HDF5-file to dict, or save dict to HDF5-file:
88
93
  ```python
89
- h5 = lazyh5('/path/to/datafile.h5')
90
- b_loaded = h5['b']
91
- g_loaded = h5['c']['g']
92
- h5.keys()
94
+ # save dict to HDF5 file:
95
+ some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
96
+ lazyh5('/path/to/datafile.h5').from_dict(some_dict)
97
+ # load dict from HDF5 file:
98
+ loaded = lazyh5('/path/to/datafile.h5').to_dict()
99
+ ```
100
+
101
+ In Jupyter, you can interactively explore the file structure:
102
+
103
+ <img width="598" alt="image" src="https://github.com/user-attachments/assets/878d4af6-ce26-4ec3-9ba5-2701fba1c07e">
104
+
105
+
106
+ -----
107
+ <details><summary>Old interface (expand this)</summary>
108
+
109
+ ```python
110
+ from daio.h5 import save_to_h5, load_from_h5
111
+ # save dict to HDF5 file:
112
+ some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
113
+ save_to_h5('/path/to/datafile.h5', some_dict)
114
+ # load dict from HDF5 file:
115
+ dict_loaded = load_from_h5('/path/to/datafile.h5')
93
116
  ```
117
+
118
+ </details>
@@ -32,7 +32,7 @@ for i in range(20):
32
32
  writer.close()
33
33
  ```
34
34
 
35
- Read video:
35
+ Read video using speed-optimized array-like indexing or iteration:
36
36
  ```python
37
37
  reader = VideoReader('/path/to/video.mp4')
38
38
  frame_7 = reader[7]
@@ -55,22 +55,48 @@ with VideoReader('/path/to/video.mp4') as reader:
55
55
 
56
56
  ### HDF5 file IO
57
57
 
58
- Write a Python dict to a HDF5 file:
58
+ Lazily load HDF5 with a dict-like interface (contents are only loaded when accessed):
59
59
  ```python
60
- from daio.h5 import save_to_h5, load_from_h5, lazyh5
61
- some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
62
- save_to_h5('/path/to/datafile.h5', some_dict)
60
+ from daio.h5 import lazyh5
61
+ h5 = lazyh5('/path/to/datafile.h5')
62
+ b_loaded = h5['b']
63
+ e_loaded = h5['c']['e']
64
+ h5.keys()
63
65
  ```
64
66
 
65
- Load HDF5 contents to dict:
67
+ Create a new HDF5 file (or add items to existing file by setting argument `readonly=False`):
66
68
  ```python
67
- dict_loaded = load_from_h5('/path/to/datafile.h5', d)
69
+ h5 = lazyh5('test.h5')
70
+ h5['a'] = 1
71
+ h5['b'] = 'hello'
72
+ h5['c'] = {} # create subgroup
73
+ h5['c']['e'] = [2,3,4]
68
74
  ```
69
75
 
70
- Lazily load HDF5 with a dict-like interface (contents are only loaded when accessed):
76
+ Load entire HDF5-file to dict, or save dict to HDF5-file:
71
77
  ```python
72
- h5 = lazyh5('/path/to/datafile.h5')
73
- b_loaded = h5['b']
74
- g_loaded = h5['c']['g']
75
- h5.keys()
78
+ # save dict to HDF5 file:
79
+ some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
80
+ lazyh5('/path/to/datafile.h5').from_dict(some_dict)
81
+ # load dict from HDF5 file:
82
+ loaded = lazyh5('/path/to/datafile.h5').to_dict()
83
+ ```
84
+
85
+ In Jupyter, you can interactively explore the file structure:
86
+
87
+ <img width="598" alt="image" src="https://github.com/user-attachments/assets/878d4af6-ce26-4ec3-9ba5-2701fba1c07e">
88
+
89
+
90
+ -----
91
+ <details><summary>Old interface (expand this)</summary>
92
+
93
+ ```python
94
+ from daio.h5 import save_to_h5, load_from_h5
95
+ # save dict to HDF5 file:
96
+ some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
97
+ save_to_h5('/path/to/datafile.h5', some_dict)
98
+ # load dict from HDF5 file:
99
+ dict_loaded = load_from_h5('/path/to/datafile.h5')
76
100
  ```
101
+
102
+ </details>
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "daio"
7
- version = "0.0.2"
7
+ version = "0.0.4"
8
8
  description = "video and data IO tools for Python"
9
9
  authors = [
10
10
  { name = "jlab.berlin" },
@@ -13,9 +13,7 @@ license = {text = "MIT"}
13
13
  readme = "README.md"
14
14
  keywords = ["python", "HDF5", "data IO", "video IO"]
15
15
  requires-python = ">=3.8"
16
- dependencies = [
17
- "numpy","tqdm","av","h5py","hdf5plugin"
18
- ]
16
+ dependencies = ["numpy","av","h5py","hdf5plugin"]
19
17
 
20
18
  [project.urls]
21
19
  Homepage = "https://github.com/danionella/daio"
@@ -0,0 +1,3 @@
1
+ """
2
+ .. include:: ../../README.md
3
+ """
@@ -1,7 +1,10 @@
1
- import json
2
- #TODO: test replacing json with orjson
3
1
  import warnings
4
- from os.path import expanduser
2
+ from os.path import expanduser, isfile, exists
3
+
4
+ try:
5
+ import orjson as json
6
+ except ImportError:
7
+ import json
5
8
 
6
9
  import numpy as np
7
10
  import h5py
@@ -9,7 +12,8 @@ import h5py
9
12
  #TODO: test defaulting to hdf5plugin.Zstd compression for json-serialized strings
10
13
 
11
14
 
12
- def save_to_h5(filename, data, serialize=True, compression=None, json_compression='gzip', verbosity=1, file_mode='w', convert_numpy_to_native=False):
15
+
16
+ def save_to_h5(filename, data, serialize=True, compression=None, json_compression='gzip', verbosity=1, file_mode='w', convert_numpy_to_native=False, h5path='/'):
13
17
  ''' Save a nested dictionary data structure to an HDF5 file.
14
18
 
15
19
  Args:
@@ -19,6 +23,8 @@ def save_to_h5(filename, data, serialize=True, compression=None, json_compressio
19
23
  compression (string): h5py compression type (e.g. 'gzip', 'lzf' or None)
20
24
  json_compression (string): h5py compression type for serialized JSON (default: 'gzip')
21
25
  file_mode (string): h5py.File access mode. 'w' (default) for create/detete and 'a' for create/append
26
+ convert_numpy_to_native (boolean): convert numpy types to native python types
27
+ h5path (string): path within the HDF5 file to save the data
22
28
 
23
29
  based on https://github.com/danionella/lib2p/blob/master/lib2putils.py
24
30
  '''
@@ -28,7 +34,7 @@ def save_to_h5(filename, data, serialize=True, compression=None, json_compressio
28
34
  for key, item in data_item.items():
29
35
  if verbosity > 1:
30
36
  print('saving entry: {} -- {}'.format(path + key, type(item)))
31
- if isinstance(item, (np.ndarray, np.int64, np.float64, str, bytes, int, float)):
37
+ if isinstance(item, (np.ndarray, np.int64, np.float64, str, bytes, int, float)) or is_h5py_compatible_array(item):
32
38
  comp = None if np.isscalar(item) else compression
33
39
  try:
34
40
  h5file[path].create_dataset(key, data=item, compression=comp)
@@ -57,10 +63,10 @@ def save_to_h5(filename, data, serialize=True, compression=None, json_compressio
57
63
 
58
64
  filename = expanduser(filename)
59
65
  with h5py.File(filename, file_mode) as h5file:
60
- recursively_save_contents_to_group(h5file, '/', data)
66
+ recursively_save_contents_to_group(h5file, h5path, data)
61
67
 
62
68
 
63
- def load_from_h5(filename):
69
+ def load_from_h5(filename, h5path='/'):
64
70
  ''' Load an HDF5 file to a dictionary
65
71
 
66
72
  Args:
@@ -95,25 +101,27 @@ def load_from_h5(filename):
95
101
 
96
102
  filename = expanduser(filename)
97
103
  with h5py.File(filename, 'r') as h5file:
98
- return recursively_load_contents_from_group(h5file, '/')
104
+ return recursively_load_contents_from_group(h5file, h5path)
99
105
 
100
106
 
101
107
  class lazyh5:
102
108
  """ A lazy-loading interface for HDF5 files.
103
109
 
104
- This class provides an easy way to access HDF5 file content without fully
105
- loading it into memory. It supports dynamic access to datasets and subgroups.
110
+ This class provides an easy way to access HDF5 file content without fully loading it into
111
+ memory or keeping the file open. It supports dynamic access to datasets and subgroups.
106
112
 
107
113
  Args:
108
114
  filepath (str): Path to the HDF5 file.
109
115
  h5path (str, optional): HDF5 group path. Defaults to '/'.
110
- overwrite (bool, optional): Whether to overwrite existing items. Defaults to False.
116
+ readonly (str, optional): Whether to open the file in read-only mode. Defaults to None (read-only iff file exists).
111
117
  """
112
118
 
113
- def __init__(self, filepath, h5path='/', overwrite=False):
119
+ def __init__(self, filepath, h5path='/', readonly=None):
114
120
  self._filepath = filepath
115
121
  self._h5path = h5path
116
- self._overwrite = overwrite
122
+ if readonly is None:
123
+ readonly = isfile(filepath)
124
+ self._readonly = readonly
117
125
 
118
126
  def keys(self):
119
127
  """Lists the keys of the current HDF5 group.
@@ -122,18 +130,64 @@ class lazyh5:
122
130
  list: List of keys in the current HDF5 group.
123
131
  """
124
132
  with h5py.File(self._filepath, 'r') as f:
125
- return list(f[self._h5path].keys())
133
+ out = list(f[self._h5path].keys())
134
+ return out
135
+
136
+ def to_dict(self):
137
+ """Reads the HDF5 file or group into a dictionary.
138
+
139
+ Returns:
140
+ dict: A dictionary representation of the HDF5 file.
141
+ """
142
+ return load_from_h5(self._filepath, h5path=self._h5path)
143
+
144
+ def from_dict(self, data):
145
+ """Writes a dictionary to the HDF5 file or group.
146
+
147
+ Args:
148
+ data (dict): A dictionary to write to the HDF5 file (if readonly is False).
149
+ """
150
+ if self._readonly:
151
+ raise ValueError("Cannot add data to a read-only lazyh5 object.")
152
+ if exists(self._filepath):
153
+ with h5py.File(self._filepath, 'r') as f:
154
+ for k, v in data.items():
155
+ if (self._h5path+'/'+k in f):
156
+ raise AttributeError(f"Dataset or group '{k}' already exists!")
157
+ save_to_h5(self._filepath, data, h5path=self._h5path, file_mode='a')
158
+
159
+ def remove_key(self, key):
160
+ """Removes the HDF5 dataset or group. Note this does not free up space in the file.
161
+
162
+ Args:
163
+ key (str): The key to remove.
164
+ """
165
+ if self._readonly:
166
+ raise ValueError("Cannot remove data from a read-only lazyh5 object.")
167
+ else:
168
+ if exists(self._filepath):
169
+ with h5py.File(self._filepath, 'a') as f:
170
+ if key in f[self._h5path]:
171
+ del f[self._h5path][key]
172
+ else:
173
+ raise KeyError(f"Key '{key}' not found in HDF5 file.")
126
174
 
127
175
  def __getitem__(self, key):
128
176
  """Gets an item by key."""
129
177
  with h5py.File(self._filepath, 'r') as f:
130
178
  item = f[self._h5path][key]
131
179
  if isinstance(item, h5py.Group):
132
- return lazyh5(self._filepath, h5path=f"{self._h5path}/{key}")
180
+ return lazyh5(self._filepath, h5path=f"{self._h5path}/{key}".lstrip('/'), readonly=self._readonly)
133
181
  elif isinstance(item, h5py.Dataset):
182
+ if h5py.check_string_dtype(item.dtype) is not None:
183
+ item = item.asstr()
134
184
  return item[()]
135
185
  else:
136
186
  raise KeyError(f"Unknown item type for key: {key}")
187
+
188
+ def __setitem__(self, key, value):
189
+ """Sets an item by key."""
190
+ self.from_dict({key: value})
137
191
 
138
192
  def __getattr__(self, key):
139
193
  """Provides dynamic attribute-style access."""
@@ -147,13 +201,8 @@ class lazyh5:
147
201
  if key.startswith('_'):
148
202
  super().__setattr__(key, value)
149
203
  return
150
-
151
- with h5py.File(self._filepath, 'a') as f:
152
- full_path = f"{self._h5path}/{key}".lstrip('/')
153
- if (full_path in f) and (not self._overwrite):
154
- raise AttributeError(f"Dataset or group '{key}' already exists.")
155
- else:
156
- f[self._h5path].create_dataset(key, data=value)
204
+
205
+ self.from_dict({key: value})
157
206
 
158
207
  def __len__(self):
159
208
  """Gets the number of items in the current HDF5 group."""
@@ -170,3 +219,56 @@ class lazyh5:
170
219
  def __dir__(self):
171
220
  """Lists all accessible attributes and keys."""
172
221
  return self.keys() + dir(super())
222
+
223
+ def inspect_structure(self, group=None, prefix=""):
224
+ """Returns a dictionary with the hierarchical structure of the HDF5 file.
225
+
226
+ Args:
227
+ group (h5py.Group, optional): The HDF5 group to start from. Defaults to the root group.
228
+ prefix (str, optional): The prefix for the keys representing the hierarchy. Defaults to "".
229
+
230
+ Returns:
231
+ dict: A dictionary where keys are paths to datasets or groups, and values are shape/dtype for datasets.
232
+ """
233
+ with h5py.File(self._filepath, 'r') as f:
234
+ # Default to the group at the current h5path
235
+ group = f[self._h5path] if group is None else group
236
+
237
+ def _recursive_structure(g):
238
+ structure = {}
239
+ for key in g:
240
+ item = g[key]
241
+ if isinstance(item, h5py.Group):
242
+ structure[key] = _recursive_structure(item)
243
+ elif isinstance(item, h5py.Dataset):
244
+ if h5py.check_string_dtype(item.dtype) is not None:
245
+ if item.dtype.itemsize <= 50:
246
+ info = item.asstr()[()]
247
+ else:
248
+ info = f"<len {item.dtype.itemsize} str>"
249
+ else:
250
+ if item.size == 1:
251
+ info = item[()]
252
+ else:
253
+ info = f"<{item.shape} {item.dtype}>"
254
+ structure[key] = info
255
+ return structure
256
+
257
+ return _recursive_structure(group)
258
+
259
+ def _ipython_display_(self):
260
+ """Displays a summary of the object in IPython."""
261
+ from IPython.display import JSON
262
+ display(JSON(self.inspect_structure(), root='/'))
263
+
264
+
265
+ def is_h5py_compatible_array(obj):
266
+ if isinstance(obj, (list, tuple)):
267
+ try:
268
+ # Ensure it can be converted to a numerical numpy array
269
+ arr = np.asarray(obj)
270
+ # Check if it is a numerical or string array (h5py supports these)
271
+ return np.issubdtype(arr.dtype, np.number) or arr.dtype.char in {'S', 'U'}
272
+ except (ValueError, TypeError):
273
+ return False
274
+ return False
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: daio
3
- Version: 0.0.2
3
+ Version: 0.0.4
4
4
  Summary: video and data IO tools for Python
5
5
  Author: jlab.berlin
6
6
  License: MIT
@@ -10,7 +10,6 @@ Requires-Python: >=3.8
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
12
  Requires-Dist: numpy
13
- Requires-Dist: tqdm
14
13
  Requires-Dist: av
15
14
  Requires-Dist: h5py
16
15
  Requires-Dist: hdf5plugin
@@ -49,7 +48,7 @@ for i in range(20):
49
48
  writer.close()
50
49
  ```
51
50
 
52
- Read video:
51
+ Read video using speed-optimized array-like indexing or iteration:
53
52
  ```python
54
53
  reader = VideoReader('/path/to/video.mp4')
55
54
  frame_7 = reader[7]
@@ -72,22 +71,48 @@ with VideoReader('/path/to/video.mp4') as reader:
72
71
 
73
72
  ### HDF5 file IO
74
73
 
75
- Write a Python dict to a HDF5 file:
74
+ Lazily load HDF5 with a dict-like interface (contents are only loaded when accessed):
76
75
  ```python
77
- from daio.h5 import save_to_h5, load_from_h5, lazyh5
78
- some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
79
- save_to_h5('/path/to/datafile.h5', some_dict)
76
+ from daio.h5 import lazyh5
77
+ h5 = lazyh5('/path/to/datafile.h5')
78
+ b_loaded = h5['b']
79
+ e_loaded = h5['c']['e']
80
+ h5.keys()
80
81
  ```
81
82
 
82
- Load HDF5 contents to dict:
83
+ Create a new HDF5 file (or add items to existing file by setting argument `readonly=False`):
83
84
  ```python
84
- dict_loaded = load_from_h5('/path/to/datafile.h5', d)
85
+ h5 = lazyh5('test.h5')
86
+ h5['a'] = 1
87
+ h5['b'] = 'hello'
88
+ h5['c'] = {} # create subgroup
89
+ h5['c']['e'] = [2,3,4]
85
90
  ```
86
91
 
87
- Lazily load HDF5 with a dict-like interface (contents are only loaded when accessed):
92
+ Load entire HDF5-file to dict, or save dict to HDF5-file:
88
93
  ```python
89
- h5 = lazyh5('/path/to/datafile.h5')
90
- b_loaded = h5['b']
91
- g_loaded = h5['c']['g']
92
- h5.keys()
94
+ # save dict to HDF5 file:
95
+ some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
96
+ lazyh5('/path/to/datafile.h5').from_dict(some_dict)
97
+ # load dict from HDF5 file:
98
+ loaded = lazyh5('/path/to/datafile.h5').to_dict()
99
+ ```
100
+
101
+ In Jupyter, you can interactively explore the file structure:
102
+
103
+ <img width="598" alt="image" src="https://github.com/user-attachments/assets/878d4af6-ce26-4ec3-9ba5-2701fba1c07e">
104
+
105
+
106
+ -----
107
+ <details><summary>Old interface (expand this)</summary>
108
+
109
+ ```python
110
+ from daio.h5 import save_to_h5, load_from_h5
111
+ # save dict to HDF5 file:
112
+ some_dict = dict(a = 1, b = np.random.randn(3,4,5), c = dict(g='nested'), d = 'some_string')
113
+ save_to_h5('/path/to/datafile.h5', some_dict)
114
+ # load dict from HDF5 file:
115
+ dict_loaded = load_from_h5('/path/to/datafile.h5')
93
116
  ```
117
+
118
+ </details>
@@ -1,6 +1,7 @@
1
1
  LICENSE
2
2
  README.md
3
3
  pyproject.toml
4
+ src/daio/__init__.py
4
5
  src/daio/h5.py
5
6
  src/daio/video.py
6
7
  src/daio.egg-info/PKG-INFO
@@ -1,5 +1,4 @@
1
1
  numpy
2
- tqdm
3
2
  av
4
3
  h5py
5
4
  hdf5plugin
File without changes
File without changes
File without changes
File without changes