hdfmap 1.3.0__tar.gz → 1.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hdfmap-1.3.0/src/hdfmap.egg-info → hdfmap-1.4.1}/PKG-INFO +1 -1
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/__init__.py +2 -2
- hdfmap-1.4.1/src/hdfmap/base_functions.py +227 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/eval_functions.py +148 -265
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/hdfmap_class.py +187 -114
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/nexus.py +91 -16
- hdfmap-1.4.1/src/hdfmap/objects.py +28 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/reloader_class.py +101 -6
- {hdfmap-1.3.0 → hdfmap-1.4.1/src/hdfmap.egg-info}/PKG-INFO +1 -1
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/SOURCES.txt +3 -0
- hdfmap-1.4.1/tests/test_depending_software.py +23 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_edge_cases.py +131 -1
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_eval_functions.py +35 -1
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_hdfmap_class.py +87 -6
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_many_files.py +38 -5
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_nexus.py +78 -1
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_reloader_class.py +8 -1
- {hdfmap-1.3.0 → hdfmap-1.4.1}/LICENSE +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/README.md +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/pyproject.toml +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/setup.cfg +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/data_holder.py +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/file_functions.py +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/hdf_loader.py +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/logging.py +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/dependency_links.txt +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/requires.txt +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/top_level.txt +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_file_functions.py +0 -0
- {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_hdf_loader.py +0 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import re
|
|
3
|
+
from types import EllipsisType
|
|
4
|
+
|
|
5
|
+
import h5py
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from hdfmap.objects import Dataset
|
|
9
|
+
from .logging import create_logger
|
|
10
|
+
|
|
11
|
+
logger = create_logger(__name__)
|
|
12
|
+
SEP = '/' # HDF path separator
|
|
13
|
+
OMIT = ['/value', '/data'] # omit these names in paths when determining identifier
|
|
14
|
+
LOCAL_NAME = 'local_name' # dataset attribute name for alt_name (DLS specific)
|
|
15
|
+
re_special_characters = re.compile(r'\W') # finds all special non-alphanumberic characters
|
|
16
|
+
datetime_converter = np.vectorize(lambda x: datetime.datetime.fromisoformat(x.decode() if hasattr(x, 'decode') else x))
|
|
17
|
+
re_long_floats = re.compile(r'\d+\.\d{5,}') # finds floats with long trailing decimals
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def generate_alt_name(hdf_dataset: h5py.Dataset) -> str | None:
|
|
21
|
+
"""Generate alt_name of dataset if 'local_name' in attributes"""
|
|
22
|
+
if LOCAL_NAME in hdf_dataset.attrs:
|
|
23
|
+
alt_name = hdf_dataset.attrs[LOCAL_NAME]
|
|
24
|
+
if hasattr(alt_name, 'decode'):
|
|
25
|
+
alt_name = alt_name.decode()
|
|
26
|
+
return expression_safe_name(alt_name.split('.')[-1])
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def generate_identifier(hdf_path: str | bytes) -> str:
|
|
31
|
+
"""
|
|
32
|
+
Generate a valid python identifier from a hdf dataset path or other string
|
|
33
|
+
- Decodes to ascii
|
|
34
|
+
- omits '/value' and '/data'
|
|
35
|
+
- splits by path separator (/) and takes final element
|
|
36
|
+
- converts special characters to '_'
|
|
37
|
+
- removes replication of strings separated by '_'
|
|
38
|
+
E.G.
|
|
39
|
+
/entry/group/motor1 >> "motor1"
|
|
40
|
+
/entry/group/motor/value >> "motor"
|
|
41
|
+
/entry/group/subgroup.motor >> "subgroup_motor"
|
|
42
|
+
motor.motor >> "motor"
|
|
43
|
+
:param hdf_path: str hdf path address
|
|
44
|
+
:return: str expression safe name
|
|
45
|
+
"""
|
|
46
|
+
if hasattr(hdf_path, 'decode'): # Byte string
|
|
47
|
+
hdf_path = hdf_path.decode('ascii')
|
|
48
|
+
for omit in OMIT:
|
|
49
|
+
hdf_path = hdf_path.removesuffix(omit) # omit 'value' or 'data'
|
|
50
|
+
substring = hdf_path.split(SEP)[-1]
|
|
51
|
+
# remove replication (handles local_names 'name.name' convention)
|
|
52
|
+
substring = '.'.join(dict.fromkeys(substring.split('.')))
|
|
53
|
+
return expression_safe_name(substring)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def build_hdf_path(*args: str | bytes) -> str:
|
|
57
|
+
"""
|
|
58
|
+
Build path from string or bytes arguments
|
|
59
|
+
'/entry/measurement' = build_hdf_path(b'entry', 'measurement')
|
|
60
|
+
:param args: str or bytes arguments
|
|
61
|
+
:return: str hdf path
|
|
62
|
+
"""
|
|
63
|
+
return SEP + SEP.join((arg.decode() if isinstance(arg, bytes) else arg).strip(SEP) for arg in args)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def expression_safe_name(string: str, replace: str = '_') -> str:
|
|
67
|
+
"""
|
|
68
|
+
Returns an expression safe name
|
|
69
|
+
:param string: any string
|
|
70
|
+
:param replace: str replace special characters with this
|
|
71
|
+
:return: string with special characters replaced
|
|
72
|
+
"""
|
|
73
|
+
return re_special_characters.sub(replace, string)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def round_string_floats(string):
|
|
77
|
+
"""
|
|
78
|
+
Shorten string by removing long floats
|
|
79
|
+
:param string: string, e.g. '#810002 scan eta 74.89533603616637 76.49533603616636 0.02 pil3_100k 1 roi2'
|
|
80
|
+
:return: shorter string, e.g. '#810002 scan eta 74.895 76.495 0.02 pil3_100k 1 roi2'
|
|
81
|
+
"""
|
|
82
|
+
def subfun(m):
|
|
83
|
+
return str(round(float(m.group()), 3))
|
|
84
|
+
return re_long_floats.sub(subfun, string)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def is_image(shape: tuple[int], min_dim=3):
|
|
88
|
+
"""Return True/False if dataset shape is suitable for image data"""
|
|
89
|
+
return len(shape) >= min_dim and (shape[-2] - 1) * (shape[-1] - 1) > 1
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def generate_image_roi_slice(start_i: int, stop_i: int, start_j: int, stop_j: int, step_i: int = 1, step_j: int = 1) -> tuple[EllipsisType, slice, slice]:
|
|
93
|
+
"""
|
|
94
|
+
Generate indexing slice for region of interest (ROI)
|
|
95
|
+
"""
|
|
96
|
+
return Ellipsis, slice(start_i, stop_i, step_i), slice(start_j, stop_j, step_j)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def make_json_serialisable(obj):
|
|
100
|
+
"""Convert h5py/numpy objects into JSON-compatible types."""
|
|
101
|
+
if isinstance(obj, bytes):
|
|
102
|
+
return obj.decode()
|
|
103
|
+
|
|
104
|
+
if isinstance(obj, np.ndarray):
|
|
105
|
+
return [make_json_serialisable(v) for v in obj.tolist()]
|
|
106
|
+
|
|
107
|
+
if isinstance(obj, np.generic):
|
|
108
|
+
return obj.item()
|
|
109
|
+
|
|
110
|
+
if isinstance(obj, dict):
|
|
111
|
+
return {k: make_json_serialisable(v) for k, v in obj.items()}
|
|
112
|
+
|
|
113
|
+
if isinstance(obj, (list, tuple)):
|
|
114
|
+
return [make_json_serialisable(v) for v in obj]
|
|
115
|
+
|
|
116
|
+
return obj
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def attrs2dict(obj: h5py.Group | h5py.Dataset) -> dict:
|
|
120
|
+
"""Extract obj.attrs as a serialisable dict"""
|
|
121
|
+
return {name: make_json_serialisable(item) for name, item in obj.attrs.items()}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def dataset2data(dataset: h5py.Dataset, index: int | slice = (), direct_load=False) -> datetime.datetime | str | np.ndarray:
|
|
125
|
+
"""
|
|
126
|
+
Read the data from a h5py Dataset and convert to either datetime, str or squeezed numpy array
|
|
127
|
+
|
|
128
|
+
- numeric size=1 datasets return float
|
|
129
|
+
- numeric size>1 datasets return numpy array
|
|
130
|
+
- string/ byte timestamps (size=1) return datetime.datetime object
|
|
131
|
+
- string/ byte timestamps (size>1) return array of datetime.datetime
|
|
132
|
+
- all other size=1 datasets return string
|
|
133
|
+
- all other size>1 datasets return array of string
|
|
134
|
+
|
|
135
|
+
:param dataset: h5py.Dataset containing data
|
|
136
|
+
:param index: index of array (not used if dataset is string/ bytes type)
|
|
137
|
+
:param direct_load: loads the data directly without conversion if True
|
|
138
|
+
:return datetime.datetime: if data is an isoformat string, returns datetime object
|
|
139
|
+
:return str: if data is another string, returns str with long floats rounded
|
|
140
|
+
:return np.ndarray: if data is another numeric object
|
|
141
|
+
"""
|
|
142
|
+
if direct_load:
|
|
143
|
+
return dataset[index]
|
|
144
|
+
if np.issubdtype(dataset, np.number):
|
|
145
|
+
logger.debug(f"Dataset {repr(dataset)} is numeric, return numpy array")
|
|
146
|
+
# note that squeeze returns float if np.float64 is given.
|
|
147
|
+
return np.squeeze(dataset[index]) # numeric np.ndarray
|
|
148
|
+
try:
|
|
149
|
+
# str integers will be cast as timestamps (years), capture as int
|
|
150
|
+
return np.squeeze(dataset[index]).astype(int)
|
|
151
|
+
except ValueError:
|
|
152
|
+
pass
|
|
153
|
+
try:
|
|
154
|
+
# timestamp -> np.datetime64 -> datetime, results in Numpy warnings for timezones, wrong time
|
|
155
|
+
# timestamp = np.squeeze(dataset[index]).astype(np.datetime64).astype(datetime.datetime)
|
|
156
|
+
timestamp = np.squeeze(datetime_converter(dataset))
|
|
157
|
+
logger.debug(f"Dataset {repr(dataset)} is timestamp, return array of datetime objects")
|
|
158
|
+
# single datetime obj vs array of datetime obj
|
|
159
|
+
return timestamp[()] if timestamp.ndim == 0 else timestamp
|
|
160
|
+
except (ValueError, OSError): # OSError arises sometimes for reasons I don't understand (e.g. old I06-1 data)
|
|
161
|
+
try:
|
|
162
|
+
string_dataset = dataset.asstr()[()]
|
|
163
|
+
logger.debug(f"Dataset {repr(dataset)} is string")
|
|
164
|
+
if dataset.ndim == 0:
|
|
165
|
+
return round_string_floats(string_dataset) # bytes or str -> str
|
|
166
|
+
return string_dataset[index] # str array
|
|
167
|
+
except ValueError:
|
|
168
|
+
logger.debug(f"Dataset {repr(dataset)} is an unexpected type")
|
|
169
|
+
return np.squeeze(dataset[index]) # other np.ndarray
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def dataset2str(dataset: h5py.Dataset, index: int | slice | tuple[slice, ...] = (), units: bool = False) -> str:
|
|
173
|
+
"""
|
|
174
|
+
Read the data from a h5py Dataset and convert to a representative string
|
|
175
|
+
|
|
176
|
+
- Strings are given with quotes
|
|
177
|
+
- long floats are shorted by attribute 'decimals'
|
|
178
|
+
- numeric arrays are summarised as "dtype (shape)"
|
|
179
|
+
- string arrays are summarised as "['str0', ...]
|
|
180
|
+
- trailing floats within strings are shortened
|
|
181
|
+
- if units=True and units available as attribute, the unit will be appended
|
|
182
|
+
|
|
183
|
+
:param dataset: h5py.Dataset containing data
|
|
184
|
+
:param index: index of array (not used if dataset is string/ bytes type)
|
|
185
|
+
:param units: if True and attribute 'units' available, append this to the result
|
|
186
|
+
:return str: string representation of data
|
|
187
|
+
"""
|
|
188
|
+
if np.issubdtype(dataset, np.number):
|
|
189
|
+
logger.debug(f"Dataset {repr(dataset)} is numeric")
|
|
190
|
+
if dataset.size > 1:
|
|
191
|
+
return f"{dataset.dtype} {dataset.shape}"
|
|
192
|
+
value = np.squeeze(dataset[index]) # size 1 numeric np.ndarray
|
|
193
|
+
if 'decimals' in dataset.attrs:
|
|
194
|
+
decimals = int(dataset.attrs['decimals'])
|
|
195
|
+
if abs(value) > 1:
|
|
196
|
+
value = value.round(decimals)
|
|
197
|
+
value = np.array2string(value, precision=decimals, separator=', ', floatmode='maxprec')
|
|
198
|
+
if units and 'units' in dataset.attrs:
|
|
199
|
+
value = f"{value} {arg.decode() if isinstance((arg := dataset.attrs['units']), bytes) else arg}"
|
|
200
|
+
return str(value)
|
|
201
|
+
try:
|
|
202
|
+
# timestamp -> datetime64 -> datetime
|
|
203
|
+
# timestamp = np.squeeze(dataset[index]).astype(np.datetime64).astype(datetime.datetime)
|
|
204
|
+
timestamp = np.squeeze(datetime_converter(dataset))
|
|
205
|
+
logger.debug(f"Dataset {repr(dataset)} is timestamp")
|
|
206
|
+
# single datetime obj vs array of datetime obj
|
|
207
|
+
return f"'{timestamp[()]}'" if timestamp.ndim == 0 else f"['{timestamp[0]}', ...({len(timestamp)})]"
|
|
208
|
+
except (ValueError, OSError): # OSError arises sometimes for reasons I don't understand (e.g. old I06-1 data)
|
|
209
|
+
try:
|
|
210
|
+
string_dataset = dataset.asstr()[()]
|
|
211
|
+
logger.debug(f"Dataset {repr(dataset)} is string")
|
|
212
|
+
if dataset.ndim == 0:
|
|
213
|
+
return f"'{round_string_floats(string_dataset)}'" # bytes or str -> str
|
|
214
|
+
return f"['{string_dataset[0]}', ...({len(string_dataset)})]" # str array
|
|
215
|
+
except ValueError:
|
|
216
|
+
logger.debug(f"Dataset {repr(dataset)} is an unexpected type")
|
|
217
|
+
return str(np.squeeze(dataset[index])) # other np.ndarray
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def dataset_attribute(dataset: h5py.Dataset | Dataset, attribute: str) -> str:
|
|
221
|
+
"""
|
|
222
|
+
Return attribute of dataset
|
|
223
|
+
"""
|
|
224
|
+
value = dataset.attrs.get(attribute, '')
|
|
225
|
+
if isinstance(value, bytes):
|
|
226
|
+
return value.decode()
|
|
227
|
+
return value
|