dataset-suite 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataset_suite-0.1.0/LICENSE +21 -0
- dataset_suite-0.1.0/PKG-INFO +11 -0
- dataset_suite-0.1.0/README.rst +21 -0
- dataset_suite-0.1.0/dataset_suite/__init__.py +652 -0
- dataset_suite-0.1.0/dataset_suite.egg-info/PKG-INFO +11 -0
- dataset_suite-0.1.0/dataset_suite.egg-info/SOURCES.txt +9 -0
- dataset_suite-0.1.0/dataset_suite.egg-info/dependency_links.txt +1 -0
- dataset_suite-0.1.0/dataset_suite.egg-info/requires.txt +3 -0
- dataset_suite-0.1.0/dataset_suite.egg-info/top_level.txt +1 -0
- dataset_suite-0.1.0/pyproject.toml +17 -0
- dataset_suite-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jakub Dranczewski
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dataset-suite
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: h5py>=3.16.0
|
|
9
|
+
Requires-Dist: matplotlib
|
|
10
|
+
Requires-Dist: numpy>2.0.0
|
|
11
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
Dataset suite (dataset-suite)
|
|
2
|
+
=============================
|
|
3
|
+
|
|
4
|
+
**Store scientific data in HDF5 files, but with pleasant
|
|
5
|
+
Python bindings!**
|
|
6
|
+
|
|
7
|
+
A general data handling toolkit with a powerful class for abstracting away work
|
|
8
|
+
with multidimensional set.
|
|
9
|
+
|
|
10
|
+
* Documentation: https://dataset-suite.readthedocs.io
|
|
11
|
+
* Repository: https://github.com/jdranczewski/dataset-suite-pip
|
|
12
|
+
|
|
13
|
+
Installation
|
|
14
|
+
------------
|
|
15
|
+
You can install this package from pip::
|
|
16
|
+
|
|
17
|
+
pip install dataset-suite
|
|
18
|
+
|
|
19
|
+
Once installed, all useful functions are in the top-level module::
|
|
20
|
+
|
|
21
|
+
import dataset_suite as ds
|
|
@@ -0,0 +1,652 @@
|
|
|
1
|
+
"""
|
|
2
|
+
A general data handling toolkit with a powerful class for abstracting away work
|
|
3
|
+
with multidimensional set. Store scientific data in HDF5 files, but with pleasant
|
|
4
|
+
Python bindings!
|
|
5
|
+
|
|
6
|
+
Code by Jakub Dranczewski
|
|
7
|
+
jdranczewski.github.io
|
|
8
|
+
jbd17@ic.ac.uk
|
|
9
|
+
jakub.dranczewski@gmail.com
|
|
10
|
+
^ one of these will work
|
|
11
|
+
|
|
12
|
+
MIT License
|
|
13
|
+
|
|
14
|
+
Copyright (c) 2026 Jakub Dranczewski
|
|
15
|
+
Parts of this code were created as part of PhD work supported by the EU ITN EID
|
|
16
|
+
project CORAL (GA no. 859841).
|
|
17
|
+
|
|
18
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
19
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
20
|
+
in the Software without restriction, including without limitation the rights
|
|
21
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
22
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
23
|
+
furnished to do so, subject to the following conditions:
|
|
24
|
+
|
|
25
|
+
The above copyright notice and this permission notice shall be included in all
|
|
26
|
+
copies or substantial portions of the Software.
|
|
27
|
+
|
|
28
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
29
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
30
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
31
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
32
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
33
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
34
|
+
SOFTWARE.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
import numpy as np
|
|
38
|
+
import os
|
|
39
|
+
import re
|
|
40
|
+
import pickle
|
|
41
|
+
import gzip
|
|
42
|
+
from glob import glob as _glob
|
|
43
|
+
import h5py
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def folders(folder="", rel_current=True):
|
|
47
|
+
"""
|
|
48
|
+
Returns a list of folders within a specified directory. Current directory by default.
|
|
49
|
+
Set rel_current to False to return just the folder names, instead of the full path
|
|
50
|
+
relative to current directory.
|
|
51
|
+
"""
|
|
52
|
+
if rel_current:
|
|
53
|
+
return [
|
|
54
|
+
os.path.join(*list(os.path.split(x)[:-1]))
|
|
55
|
+
for x in glob(os.path.join(folder, "*", ""))
|
|
56
|
+
]
|
|
57
|
+
else:
|
|
58
|
+
return [os.path.split(x)[-2] for x in glob(os.path.join(folder, "*", ""))]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def prefixes(folder, split, yes=[], no=[]):
|
|
62
|
+
"""
|
|
63
|
+
Returns a list of file prefixes in a given folder that match some conditions.
|
|
64
|
+
'prefix' is defined as the part of the filename before the string given as 'split'.
|
|
65
|
+
'yes' and 'no' are lists/tuples or string that should/should not be in the full file
|
|
66
|
+
name, so act as filters.
|
|
67
|
+
"""
|
|
68
|
+
files = os.listdir(folder)
|
|
69
|
+
for s in yes + [split]:
|
|
70
|
+
files = [x for x in files if s in x]
|
|
71
|
+
for s in no:
|
|
72
|
+
files = [x for x in files if s not in x]
|
|
73
|
+
prefixes = list(set([x.split(split)[0] for x in files]))
|
|
74
|
+
prefixes.sort()
|
|
75
|
+
return prefixes
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def glob(path, yes=[], no=[]):
|
|
79
|
+
"""
|
|
80
|
+
A small extension to the excellent glob library. 'path' is defined as for the usual
|
|
81
|
+
glob: a Unix-style pathname with * as a wildcard.
|
|
82
|
+
'yes' and 'no' are lists/tuples or string that should/should not be in the full file
|
|
83
|
+
name, so act as filters.
|
|
84
|
+
"""
|
|
85
|
+
files = _glob(path)
|
|
86
|
+
for s in yes:
|
|
87
|
+
files = [x for x in files if s in x]
|
|
88
|
+
for s in no:
|
|
89
|
+
files = [x for x in files if s not in x]
|
|
90
|
+
files.sort()
|
|
91
|
+
return files
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def extract(string, *patterns):
|
|
95
|
+
"""
|
|
96
|
+
Given a string and an arbitrary number of patterns as arguments, this function extracts
|
|
97
|
+
associated numbers from the string. For example, '_P_' as a pattern will match '_P_123'
|
|
98
|
+
in a filename and return 123.
|
|
99
|
+
"""
|
|
100
|
+
return [
|
|
101
|
+
int(re.search(pattern + "([0-9]+)", string).group(1)) for pattern in patterns
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def extract_raw(string, *patterns):
|
|
106
|
+
"""
|
|
107
|
+
You can write regex? Good for you.
|
|
108
|
+
Given a string and an arbitrary number of patterns as arguments, this function extracts
|
|
109
|
+
associated things from the string. You have to include a capture group, otherwise errors ensue.
|
|
110
|
+
"""
|
|
111
|
+
return [re.search(pattern, string).group(1) for pattern in patterns]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def extract_unique(files, pattern):
|
|
115
|
+
numbers = list(set([extract(x, pattern)[0] for x in files]))
|
|
116
|
+
numbers.sort()
|
|
117
|
+
return numbers
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _sort_key(x, pattern):
|
|
121
|
+
try:
|
|
122
|
+
return int(re.search(pattern + "([0-9]+)", x).group(1))
|
|
123
|
+
except AttributeError:
|
|
124
|
+
return 1e10
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def sort_by(files, pattern):
|
|
128
|
+
"""
|
|
129
|
+
Given a list of strings (like file names or sth) and a pattern, the functions extracts
|
|
130
|
+
an associated numeric parameter from each string and sorts the list according to that
|
|
131
|
+
parameter.
|
|
132
|
+
|
|
133
|
+
For example, if your files contain power information as '_P_123', pass '_P_' as 'pattern'.
|
|
134
|
+
Sorting happens in-place in the passed list, so nothing is returned.
|
|
135
|
+
"""
|
|
136
|
+
files.sort(key=lambda x: _sort_key(x, pattern))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def colours(values, cmap: str | None = None, minmax=None):
|
|
140
|
+
# Check if matplotlib needs to be imported.
|
|
141
|
+
# We only do this here, as this takes a bit of time, so it's silly to do this
|
|
142
|
+
# every time the full library is imported.
|
|
143
|
+
if "matplotlib" not in globals():
|
|
144
|
+
global matplotlib
|
|
145
|
+
import matplotlib
|
|
146
|
+
|
|
147
|
+
# Default colourmap
|
|
148
|
+
if cmap is None:
|
|
149
|
+
cmap = "viridis"
|
|
150
|
+
cmap_obj = matplotlib.colormaps[cmap]
|
|
151
|
+
|
|
152
|
+
if minmax is None:
|
|
153
|
+
a, b = np.amin(values), np.amax(values)
|
|
154
|
+
else:
|
|
155
|
+
a, b = minmax
|
|
156
|
+
|
|
157
|
+
if len(values) == 1:
|
|
158
|
+
return cmap_obj((0,))
|
|
159
|
+
return cmap_obj((values - a) / (b - a))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def map_axes(data, **axes):
|
|
163
|
+
"""
|
|
164
|
+
Helper function for figuring out how the axes you have map to the data.
|
|
165
|
+
data is the ndarray, and then you pass your axes in any order as keyword arguments.
|
|
166
|
+
"""
|
|
167
|
+
data = np.asarray(data)
|
|
168
|
+
shape = data.shape
|
|
169
|
+
for s in shape:
|
|
170
|
+
print(s, [key for key in axes if len(axes[key]) == s])
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def load(filename):
|
|
174
|
+
"""
|
|
175
|
+
Load a dataset-suite object saved as a pickle with `save`.
|
|
176
|
+
"""
|
|
177
|
+
with gzip.open(filename, "rb") as f:
|
|
178
|
+
return pickle.load(f)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def load_h5(filename):
|
|
182
|
+
"""
|
|
183
|
+
Load a dataset-suite object saved as a HDF5 file with `save_h5`.
|
|
184
|
+
"""
|
|
185
|
+
with h5py.File(filename, "r") as h5:
|
|
186
|
+
return _from_h5_router(h5)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _dict_to_h5(
|
|
190
|
+
h5: h5py.File | h5py.Group, name: str, dictionary: dict, compression: int
|
|
191
|
+
) -> None:
|
|
192
|
+
group: h5py.Group = h5.create_group(name, track_order=True)
|
|
193
|
+
for key in dictionary:
|
|
194
|
+
_to_h5_router(group, key, dictionary[key], compression)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _h5_to_dict(h5: h5py.Group) -> dict:
|
|
198
|
+
out = {}
|
|
199
|
+
for key in h5:
|
|
200
|
+
data = h5[key][...]
|
|
201
|
+
if len(data.shape) == 0:
|
|
202
|
+
data = data[()]
|
|
203
|
+
if isinstance(data, bytes):
|
|
204
|
+
data = data.decode("utf-8")
|
|
205
|
+
out[key] = data
|
|
206
|
+
return out
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _to_h5_router(
|
|
210
|
+
group: h5py.File | h5py.Group, key: str, data, compression: int
|
|
211
|
+
) -> h5py.Group | h5py.Dataset:
|
|
212
|
+
if hasattr(data, "_populate_h5"):
|
|
213
|
+
sub_group: h5py.Group = group.create_group(key, track_order=True)
|
|
214
|
+
data._populate_h5(sub_group, compression)
|
|
215
|
+
return sub_group
|
|
216
|
+
else:
|
|
217
|
+
try:
|
|
218
|
+
# Is it a numpy array, or something that can be cast to a homogeneous numpy array?
|
|
219
|
+
sub_group: h5py.Dataset = group.create_dataset(
|
|
220
|
+
key, data=data, compression="gzip", compression_opts=compression
|
|
221
|
+
)
|
|
222
|
+
return sub_group
|
|
223
|
+
except (TypeError, ValueError):
|
|
224
|
+
# Is it a scalar that can be stored as an un-compressed dataset?
|
|
225
|
+
try:
|
|
226
|
+
# Special case for strings
|
|
227
|
+
# https://docs.h5py.org/en/latest/strings.html
|
|
228
|
+
if isinstance(data, np.ndarray) and "U" in str(data.dtype):
|
|
229
|
+
data = str(data)
|
|
230
|
+
sub_group: h5py.Dataset = group.create_dataset(
|
|
231
|
+
key,
|
|
232
|
+
data=data,
|
|
233
|
+
)
|
|
234
|
+
return sub_group
|
|
235
|
+
except (TypeError, ValueError):
|
|
236
|
+
# Is it a list or tuple?
|
|
237
|
+
try:
|
|
238
|
+
iterator = enumerate(data)
|
|
239
|
+
except TypeError:
|
|
240
|
+
raise TypeError(
|
|
241
|
+
f"Couldn't store data in an h5 file at {group} / {key}"
|
|
242
|
+
)
|
|
243
|
+
sub_group: h5py.Group = group.create_group(key, track_order=True)
|
|
244
|
+
sub_group.attrs["dataset_type"] = "list"
|
|
245
|
+
for i, sub_data in iterator:
|
|
246
|
+
_to_h5_router(sub_group, str(i), sub_data, compression)
|
|
247
|
+
return sub_group
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _from_h5_router(h5: h5py.File | h5py.Group | h5py.Dataset):
|
|
251
|
+
if isinstance(h5, h5py.Dataset):
|
|
252
|
+
value = h5[...]
|
|
253
|
+
if not len(value.shape):
|
|
254
|
+
# just a single value rather than an array
|
|
255
|
+
value = value[()]
|
|
256
|
+
return value
|
|
257
|
+
if "dataset_type" not in h5.attrs:
|
|
258
|
+
raise ValueError("This h5 object does not conform to dataset-suite standards.")
|
|
259
|
+
if h5.attrs["dataset_type"] == "dataset":
|
|
260
|
+
return dataset.from_h5(h5)
|
|
261
|
+
elif h5.attrs["dataset_type"] == "datalist":
|
|
262
|
+
return datalist.from_h5(h5)
|
|
263
|
+
elif h5.attrs["dataset_type"] == "datadict":
|
|
264
|
+
return datadict.from_h5(h5)
|
|
265
|
+
elif h5.attrs["dataset_type"] == "list":
|
|
266
|
+
return [_from_h5_router(h5[key]) for key in h5.keys()]
|
|
267
|
+
raise ValueError(f"No valid dataset-suite object type found for {h5}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class _base_dataobject:
|
|
271
|
+
def save(self, filename, compress=6) -> None:
|
|
272
|
+
with gzip.open(filename, "wb", compresslevel=compress) as f:
|
|
273
|
+
pickle.dump(self, f)
|
|
274
|
+
|
|
275
|
+
def save_h5(self, filename: str, compress: int = 6) -> None:
|
|
276
|
+
with h5py.File(filename, "w", track_order=True) as f:
|
|
277
|
+
self._populate_h5(f, compress)
|
|
278
|
+
# It's possible that the _populate call above didn't create a metadata group, check and create
|
|
279
|
+
# so we can write the filename into it
|
|
280
|
+
meta_group = (
|
|
281
|
+
f["metadata"]
|
|
282
|
+
if "metadata" in f.keys()
|
|
283
|
+
else f.create_group("metadata", track_order=True)
|
|
284
|
+
)
|
|
285
|
+
meta_group.create_dataset("filename", data=os.path.basename(filename))
|
|
286
|
+
|
|
287
|
+
@classmethod
|
|
288
|
+
def from_h5(cls, h5: h5py.File | h5py.Group):
|
|
289
|
+
raise NotImplementedError
|
|
290
|
+
|
|
291
|
+
def _populate_h5(self, h5: h5py.File | h5py.Group, compression: int) -> None:
|
|
292
|
+
raise NotImplementedError
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class dataset(_base_dataobject):
|
|
296
|
+
"""
|
|
297
|
+
A container for data arrays with labelled axes.
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
def __init__(self, data, cut=None, **axes):
|
|
301
|
+
self._raw = np.asarray(data)
|
|
302
|
+
if len(axes) != self._raw.ndim:
|
|
303
|
+
raise IndexError("The number of provided axes does not match the dataset.")
|
|
304
|
+
for i, key in enumerate(axes):
|
|
305
|
+
if self._raw.shape[i] != len(axes[key]):
|
|
306
|
+
raise IndexError(
|
|
307
|
+
"The shape of the provided axes does not match the dataset."
|
|
308
|
+
)
|
|
309
|
+
setattr(self, key, np.asarray(axes[key]))
|
|
310
|
+
self._axes = list(axes.keys())
|
|
311
|
+
if cut is None:
|
|
312
|
+
cut = {}
|
|
313
|
+
self._cut = cut
|
|
314
|
+
self.metadata = {}
|
|
315
|
+
|
|
316
|
+
def take(self, **i):
|
|
317
|
+
"""
|
|
318
|
+
Return a slice of the dataset at the specified index for a given axis (as a dataset).
|
|
319
|
+
|
|
320
|
+
For example::
|
|
321
|
+
|
|
322
|
+
data.take(power=2)
|
|
323
|
+
"""
|
|
324
|
+
s_raw = self._raw
|
|
325
|
+
cut = self._cut
|
|
326
|
+
new_axes = self.ax_dict
|
|
327
|
+
new_ax_names = self.axes.copy()
|
|
328
|
+
for key in i:
|
|
329
|
+
s_raw = np.moveaxis(s_raw, new_ax_names.index(key), 0)[i[key]]
|
|
330
|
+
cut[key] = self.axis(key)[i[key]]
|
|
331
|
+
new_axes.pop(key)
|
|
332
|
+
new_ax_names.remove(key)
|
|
333
|
+
new_data = dataset(s_raw, cut=cut, **new_axes)
|
|
334
|
+
if hasattr(self, "metadata"):
|
|
335
|
+
new_data.metadata = self.metadata
|
|
336
|
+
return new_data
|
|
337
|
+
|
|
338
|
+
def take_raw(self, **i):
|
|
339
|
+
"""
|
|
340
|
+
Return a slice of the dataset at the specified index for a given axis (as a numpy array).
|
|
341
|
+
|
|
342
|
+
For example::
|
|
343
|
+
|
|
344
|
+
data.take_raw(power=2)
|
|
345
|
+
# equivalent to:
|
|
346
|
+
data.take(power=2).raw
|
|
347
|
+
"""
|
|
348
|
+
s_raw = self._raw
|
|
349
|
+
new_ax_names = self.axes.copy()
|
|
350
|
+
for key in i:
|
|
351
|
+
s_raw = np.moveaxis(s_raw, new_ax_names.index(key), 0)[i[key]]
|
|
352
|
+
new_ax_names.remove(key)
|
|
353
|
+
return s_raw
|
|
354
|
+
|
|
355
|
+
def take_sum(self, axis):
|
|
356
|
+
"""
|
|
357
|
+
Sum this dataset along the named axis and return the resulting dataset.
|
|
358
|
+
|
|
359
|
+
For example::
|
|
360
|
+
|
|
361
|
+
data.take_sum("wavelength")
|
|
362
|
+
"""
|
|
363
|
+
new_axes = self.ax_dict
|
|
364
|
+
new_axes.pop(axis)
|
|
365
|
+
new_data = dataset(np.sum(self._raw, axis=self._axes.index(axis)), **new_axes)
|
|
366
|
+
if hasattr(self, "metadata"):
|
|
367
|
+
new_data.metadata = self.metadata
|
|
368
|
+
return new_data
|
|
369
|
+
|
|
370
|
+
def expand(self, new_axis, value):
|
|
371
|
+
return dataset(
|
|
372
|
+
np.expand_dims(self._raw, axis=0), **{new_axis: [value], **self.ax_dict}
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
def join(self, other, axis):
|
|
376
|
+
new_axes = self.ax_dict
|
|
377
|
+
new_axes[axis] = np.concatenate((self.axis(axis), other.axis(axis)))
|
|
378
|
+
return dataset(
|
|
379
|
+
np.concatenate((self._raw, other.raw), axis=self._axes.index(axis)),
|
|
380
|
+
**new_axes,
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
def axis(self, ax):
|
|
384
|
+
return getattr(self, ax)
|
|
385
|
+
|
|
386
|
+
def astype(self, _type):
|
|
387
|
+
new_data = dataset(self._raw.astype(_type), cut=self._cut, **self.ax_dict)
|
|
388
|
+
new_data.metadata = self.metadata
|
|
389
|
+
return new_data
|
|
390
|
+
|
|
391
|
+
@property
|
|
392
|
+
def raw(self):
|
|
393
|
+
return self._raw
|
|
394
|
+
|
|
395
|
+
@property
|
|
396
|
+
def axes(self):
|
|
397
|
+
return self._axes
|
|
398
|
+
|
|
399
|
+
@property
|
|
400
|
+
def ax_dict(self):
|
|
401
|
+
return {key: self.axis(key) for key in self._axes}
|
|
402
|
+
|
|
403
|
+
@property
|
|
404
|
+
def cut(self):
|
|
405
|
+
return self._cut
|
|
406
|
+
|
|
407
|
+
def add_cut(self, key, value):
|
|
408
|
+
self._cut[key] = value
|
|
409
|
+
|
|
410
|
+
def _populate_h5(self, h5: h5py.File | h5py.Group, compression: int) -> None:
|
|
411
|
+
h5.attrs["dataset_type"] = "dataset"
|
|
412
|
+
|
|
413
|
+
data_group: h5py.Group = h5.create_group("data", track_order=True)
|
|
414
|
+
data_group.create_dataset(
|
|
415
|
+
"values", data=self.raw, compression="gzip", compression_opts=compression
|
|
416
|
+
)
|
|
417
|
+
data_group.attrs["NX_class"] = "NXdata"
|
|
418
|
+
data_group.attrs["signal"] = "values"
|
|
419
|
+
|
|
420
|
+
# Add the axes to the main data group
|
|
421
|
+
axes = []
|
|
422
|
+
for key in self.ax_dict:
|
|
423
|
+
data_group.create_dataset(
|
|
424
|
+
key,
|
|
425
|
+
data=self.ax_dict[key],
|
|
426
|
+
compression="gzip",
|
|
427
|
+
compression_opts=compression,
|
|
428
|
+
)
|
|
429
|
+
axes.append(key)
|
|
430
|
+
data_group.attrs["axes"] = axes
|
|
431
|
+
|
|
432
|
+
# Store metadata
|
|
433
|
+
_dict_to_h5(h5, "metadata", self.metadata, compression)
|
|
434
|
+
_dict_to_h5(h5, "cut", self.cut, compression)
|
|
435
|
+
|
|
436
|
+
@classmethod
|
|
437
|
+
def from_h5(cls, h5: h5py.File | h5py.Group):
|
|
438
|
+
axes = {
|
|
439
|
+
axis_key: h5["data"][axis_key][...] for axis_key in h5["data"].attrs["axes"]
|
|
440
|
+
}
|
|
441
|
+
obj = cls(h5["data"]["values"], _h5_to_dict(h5["cut"]), **axes)
|
|
442
|
+
obj.metadata = _h5_to_dict(h5["metadata"])
|
|
443
|
+
return obj
|
|
444
|
+
|
|
445
|
+
def __repr__(self):
|
|
446
|
+
return "dataset({})".format(
|
|
447
|
+
", ".join("{}[{}]".format(key, len(self.axis(key))) for key in self._axes)
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
class datalist(_base_dataobject):
|
|
452
|
+
"""
|
|
453
|
+
A container for data, like a list, but with an axis array that can contain values.
|
|
454
|
+
|
|
455
|
+
For example, this could store a dataset for each stage position in a scan (but generally
|
|
456
|
+
you should consider if using a dataset directly may not be better).
|
|
457
|
+
"""
|
|
458
|
+
|
|
459
|
+
def __init__(self, axis, cut=None):
|
|
460
|
+
self._axes = [
|
|
461
|
+
axis,
|
|
462
|
+
]
|
|
463
|
+
self._axis = []
|
|
464
|
+
self._datasets = []
|
|
465
|
+
setattr(self, axis, self._axis)
|
|
466
|
+
if cut is None:
|
|
467
|
+
cut = {}
|
|
468
|
+
self._cut = cut
|
|
469
|
+
self.metadata = {}
|
|
470
|
+
|
|
471
|
+
def append(self, ds, value):
|
|
472
|
+
try:
|
|
473
|
+
ds.add_cut(self._axes[0], value)
|
|
474
|
+
except AttributeError:
|
|
475
|
+
pass
|
|
476
|
+
self._datasets.append(ds)
|
|
477
|
+
self._axis.append(value)
|
|
478
|
+
|
|
479
|
+
def add_cut(self, key, value):
|
|
480
|
+
self._cut[key] = value
|
|
481
|
+
for ds in self:
|
|
482
|
+
try:
|
|
483
|
+
ds.add_cut(key, value)
|
|
484
|
+
except AttributeError:
|
|
485
|
+
pass
|
|
486
|
+
|
|
487
|
+
@property
|
|
488
|
+
def axis(self):
|
|
489
|
+
return self._axis
|
|
490
|
+
|
|
491
|
+
@property
|
|
492
|
+
def axes(self):
|
|
493
|
+
return self._axes
|
|
494
|
+
|
|
495
|
+
@property
|
|
496
|
+
def datasets(self):
|
|
497
|
+
return self._datasets
|
|
498
|
+
|
|
499
|
+
@property
|
|
500
|
+
def cut(self):
|
|
501
|
+
return self._cut
|
|
502
|
+
|
|
503
|
+
def _populate_h5(self, h5: h5py.File | h5py.Group, compression: int) -> None:
|
|
504
|
+
h5.attrs["dataset_type"] = "datalist"
|
|
505
|
+
|
|
506
|
+
data_group: h5py.Group = h5.create_group("data", track_order=True)
|
|
507
|
+
data_group.attrs["axes"] = self.axes
|
|
508
|
+
_to_h5_router(data_group, self.axes[0], self.axis, compression)
|
|
509
|
+
for i, value, data in zip(range(len(self.axis)), self.axis, self._datasets):
|
|
510
|
+
group = _to_h5_router(data_group, str(i), value, compression)
|
|
511
|
+
group.attrs["axis_value"] = value
|
|
512
|
+
|
|
513
|
+
# Store metadata
|
|
514
|
+
_dict_to_h5(h5, "metadata", self.metadata, compression)
|
|
515
|
+
_dict_to_h5(h5, "cut", self.cut, compression)
|
|
516
|
+
|
|
517
|
+
@classmethod
|
|
518
|
+
def from_h5(cls, h5: h5py.File | h5py.Group):
|
|
519
|
+
axis = h5["data"].attrs["axes"][0]
|
|
520
|
+
obj = cls(axis, _h5_to_dict(h5["cut"]))
|
|
521
|
+
obj.metadata = _h5_to_dict(h5["metadata"])
|
|
522
|
+
axis_values = h5["data"][axis][...]
|
|
523
|
+
for i, axis_value in enumerate(axis_values):
|
|
524
|
+
obj.append(_from_h5_router(h5["data"][str(i)]), axis_value)
|
|
525
|
+
return obj
|
|
526
|
+
|
|
527
|
+
def __getitem__(self, i: int):
|
|
528
|
+
return self._datasets[i]
|
|
529
|
+
|
|
530
|
+
def __len__(self):
|
|
531
|
+
return len(self._datasets)
|
|
532
|
+
|
|
533
|
+
def __repr__(self):
|
|
534
|
+
return str(self._datasets)
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
class datadict(_base_dataobject):
|
|
538
|
+
"""
|
|
539
|
+
A key: value store for data like a Python dictionary, but with the metadata, cut, and save mechanics
|
|
540
|
+
from the dataset object.
|
|
541
|
+
"""
|
|
542
|
+
|
|
543
|
+
def __init__(self, name, cut=None):
|
|
544
|
+
self._name = name
|
|
545
|
+
self._dict = {}
|
|
546
|
+
if cut is None:
|
|
547
|
+
cut = {}
|
|
548
|
+
self._cut = cut
|
|
549
|
+
self.metadata = {}
|
|
550
|
+
|
|
551
|
+
def add_cut(self, key, value):
|
|
552
|
+
self._cut[key] = value
|
|
553
|
+
for element in self:
|
|
554
|
+
try:
|
|
555
|
+
self._dict[element].add_cut(key, value)
|
|
556
|
+
except AttributeError:
|
|
557
|
+
pass
|
|
558
|
+
|
|
559
|
+
@property
|
|
560
|
+
def name(self):
|
|
561
|
+
return self._name
|
|
562
|
+
|
|
563
|
+
@property
|
|
564
|
+
def dict(self):
|
|
565
|
+
return self._dict
|
|
566
|
+
|
|
567
|
+
@property
|
|
568
|
+
def cut(self):
|
|
569
|
+
return self._cut
|
|
570
|
+
|
|
571
|
+
def __setitem__(self, key, value):
|
|
572
|
+
try:
|
|
573
|
+
value.add_cut(self._name, key)
|
|
574
|
+
except AttributeError:
|
|
575
|
+
pass
|
|
576
|
+
|
|
577
|
+
self._dict[key] = value
|
|
578
|
+
|
|
579
|
+
def __iter__(self):
|
|
580
|
+
for key in self._dict:
|
|
581
|
+
yield key
|
|
582
|
+
|
|
583
|
+
def __getitem__(self, key):
|
|
584
|
+
return self._dict[key]
|
|
585
|
+
|
|
586
|
+
def __contains__(self, item):
|
|
587
|
+
return item in self._dict
|
|
588
|
+
|
|
589
|
+
def keys(self):
|
|
590
|
+
return self._dict.keys()
|
|
591
|
+
|
|
592
|
+
def _populate_h5(self, h5: h5py.File | h5py.Group, compression: int) -> None:
|
|
593
|
+
h5.attrs["dataset_type"] = "datadict"
|
|
594
|
+
h5.attrs["datadict_name"] = self.name
|
|
595
|
+
|
|
596
|
+
data_group: h5py.Group = h5.create_group("data", track_order=True)
|
|
597
|
+
for key in self:
|
|
598
|
+
data = self[key]
|
|
599
|
+
_to_h5_router(data_group, key, data, compression)
|
|
600
|
+
|
|
601
|
+
# Store metadata
|
|
602
|
+
_dict_to_h5(h5, "metadata", self.metadata, compression)
|
|
603
|
+
_dict_to_h5(h5, "cut", self.cut, compression)
|
|
604
|
+
|
|
605
|
+
@classmethod
|
|
606
|
+
def from_h5(cls, h5: h5py.File | h5py.Group):
|
|
607
|
+
obj = cls(h5.attrs["datadict_name"], _h5_to_dict(h5["cut"]))
|
|
608
|
+
obj.metadata = _h5_to_dict(h5["metadata"])
|
|
609
|
+
for key in h5["data"]:
|
|
610
|
+
obj[key] = _from_h5_router(h5["data"][key])
|
|
611
|
+
return obj
|
|
612
|
+
|
|
613
|
+
def __repr__(self):
|
|
614
|
+
return "datadict({}: {})".format(self._name, ", ".join(self._dict.keys()))
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
# Printing a datadict
|
|
618
|
+
def print_value(value, offset=0):
|
|
619
|
+
"""
|
|
620
|
+
Print a value with standard formatting. Used in ``print_dict``.
|
|
621
|
+
"""
|
|
622
|
+
if isinstance(value, np.ndarray):
|
|
623
|
+
if len(value.shape):
|
|
624
|
+
print(f"array({value.shape})", end="")
|
|
625
|
+
else:
|
|
626
|
+
print(value, end="")
|
|
627
|
+
elif isinstance(value, datadict) or isinstance(value, dict):
|
|
628
|
+
print("")
|
|
629
|
+
print_dict(value, offset + 1)
|
|
630
|
+
elif isinstance(value, tuple) or isinstance(value, list):
|
|
631
|
+
print("(", end="")
|
|
632
|
+
for subvalue in value:
|
|
633
|
+
print_value(subvalue, offset)
|
|
634
|
+
print(", ", end="")
|
|
635
|
+
print(")", end="")
|
|
636
|
+
else:
|
|
637
|
+
rep = str(value)
|
|
638
|
+
if len(rep) > 60:
|
|
639
|
+
rep = f"{rep[:29]}..{rep[-29:]}"
|
|
640
|
+
print(rep, end="")
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def print_dict(data: dict, offset=0):
|
|
644
|
+
"""
|
|
645
|
+
Print a nicely formatted view of the data in a nested
|
|
646
|
+
dictionary/datadict.
|
|
647
|
+
"""
|
|
648
|
+
for key in data:
|
|
649
|
+
print(f"{offset*4*' '}{key}: ", end="")
|
|
650
|
+
value = data[key]
|
|
651
|
+
print_value(value, offset)
|
|
652
|
+
print("")
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dataset-suite
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: h5py>=3.16.0
|
|
9
|
+
Requires-Dist: matplotlib
|
|
10
|
+
Requires-Dist: numpy>2.0.0
|
|
11
|
+
Dynamic: license-file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dataset_suite
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dataset-suite"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Add your description here"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"h5py>=3.16.0",
|
|
9
|
+
"matplotlib",
|
|
10
|
+
"numpy>2.0.0",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
[dependency-groups]
|
|
14
|
+
dev = [
|
|
15
|
+
"sphinx>=8.1.3",
|
|
16
|
+
"sphinx-rtd-theme>=3.1.0",
|
|
17
|
+
]
|