hdfmap 1.3.0__tar.gz → 1.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {hdfmap-1.3.0/src/hdfmap.egg-info → hdfmap-1.4.1}/PKG-INFO +1 -1
  2. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/__init__.py +2 -2
  3. hdfmap-1.4.1/src/hdfmap/base_functions.py +227 -0
  4. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/eval_functions.py +148 -265
  5. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/hdfmap_class.py +187 -114
  6. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/nexus.py +91 -16
  7. hdfmap-1.4.1/src/hdfmap/objects.py +28 -0
  8. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/reloader_class.py +101 -6
  9. {hdfmap-1.3.0 → hdfmap-1.4.1/src/hdfmap.egg-info}/PKG-INFO +1 -1
  10. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/SOURCES.txt +3 -0
  11. hdfmap-1.4.1/tests/test_depending_software.py +23 -0
  12. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_edge_cases.py +131 -1
  13. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_eval_functions.py +35 -1
  14. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_hdfmap_class.py +87 -6
  15. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_many_files.py +38 -5
  16. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_nexus.py +78 -1
  17. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_reloader_class.py +8 -1
  18. {hdfmap-1.3.0 → hdfmap-1.4.1}/LICENSE +0 -0
  19. {hdfmap-1.3.0 → hdfmap-1.4.1}/README.md +0 -0
  20. {hdfmap-1.3.0 → hdfmap-1.4.1}/pyproject.toml +0 -0
  21. {hdfmap-1.3.0 → hdfmap-1.4.1}/setup.cfg +0 -0
  22. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/data_holder.py +0 -0
  23. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/file_functions.py +0 -0
  24. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/hdf_loader.py +0 -0
  25. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap/logging.py +0 -0
  26. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/dependency_links.txt +0 -0
  27. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/requires.txt +0 -0
  28. {hdfmap-1.3.0 → hdfmap-1.4.1}/src/hdfmap.egg-info/top_level.txt +0 -0
  29. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_file_functions.py +0 -0
  30. {hdfmap-1.3.0 → hdfmap-1.4.1}/tests/test_hdf_loader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hdfmap
3
- Version: 1.3.0
3
+ Version: 1.4.1
4
4
  Summary: Map objects within a HDF5 file and create a dataset namespace
5
5
  Author-email: Dan Porter <dan.porter@diamond.ac.uk>
6
6
  Maintainer-email: Dan Porter <dan.porter@diamond.ac.uk>
@@ -66,8 +66,8 @@ __all__ = [
66
66
  'set_all_logging_level', 'version_info', 'module_info'
67
67
  ]
68
68
 
69
- __version__ = "1.3.0"
70
- __date__ = "2026/08/14"
69
+ __version__ = "1.4.1"
70
+ __date__ = "2026/11/16"
71
71
  __author__ = "Dan Porter"
72
72
 
73
73
 
@@ -0,0 +1,227 @@
1
+ import datetime
2
+ import re
3
+ from types import EllipsisType
4
+
5
+ import h5py
6
+ import numpy as np
7
+
8
+ from hdfmap.objects import Dataset
9
+ from .logging import create_logger
10
+
11
+ logger = create_logger(__name__)
12
+ SEP = '/' # HDF path separator
13
+ OMIT = ['/value', '/data'] # omit these names in paths when determining identifier
14
+ LOCAL_NAME = 'local_name' # dataset attribute name for alt_name (DLS specific)
15
+ re_special_characters = re.compile(r'\W') # finds all special non-alphanumberic characters
16
+ datetime_converter = np.vectorize(lambda x: datetime.datetime.fromisoformat(x.decode() if hasattr(x, 'decode') else x))
17
+ re_long_floats = re.compile(r'\d+\.\d{5,}') # finds floats with long trailing decimals
18
+
19
+
20
+ def generate_alt_name(hdf_dataset: h5py.Dataset) -> str | None:
21
+ """Generate alt_name of dataset if 'local_name' in attributes"""
22
+ if LOCAL_NAME in hdf_dataset.attrs:
23
+ alt_name = hdf_dataset.attrs[LOCAL_NAME]
24
+ if hasattr(alt_name, 'decode'):
25
+ alt_name = alt_name.decode()
26
+ return expression_safe_name(alt_name.split('.')[-1])
27
+ return None
28
+
29
+
30
+ def generate_identifier(hdf_path: str | bytes) -> str:
31
+ """
32
+ Generate a valid python identifier from a hdf dataset path or other string
33
+ - Decodes to ascii
34
+ - omits '/value' and '/data'
35
+ - splits by path separator (/) and takes final element
36
+ - converts special characters to '_'
37
+ - removes replication of strings separated by '_'
38
+ E.G.
39
+ /entry/group/motor1 >> "motor1"
40
+ /entry/group/motor/value >> "motor"
41
+ /entry/group/subgroup.motor >> "subgroup_motor"
42
+ motor.motor >> "motor"
43
+ :param hdf_path: str hdf path address
44
+ :return: str expression safe name
45
+ """
46
+ if hasattr(hdf_path, 'decode'): # Byte string
47
+ hdf_path = hdf_path.decode('ascii')
48
+ for omit in OMIT:
49
+ hdf_path = hdf_path.removesuffix(omit) # omit 'value' or 'data'
50
+ substring = hdf_path.split(SEP)[-1]
51
+ # remove replication (handles local_names 'name.name' convention)
52
+ substring = '.'.join(dict.fromkeys(substring.split('.')))
53
+ return expression_safe_name(substring)
54
+
55
+
56
+ def build_hdf_path(*args: str | bytes) -> str:
57
+ """
58
+ Build path from string or bytes arguments
59
+ '/entry/measurement' = build_hdf_path(b'entry', 'measurement')
60
+ :param args: str or bytes arguments
61
+ :return: str hdf path
62
+ """
63
+ return SEP + SEP.join((arg.decode() if isinstance(arg, bytes) else arg).strip(SEP) for arg in args)
64
+
65
+
66
+ def expression_safe_name(string: str, replace: str = '_') -> str:
67
+ """
68
+ Returns an expression safe name
69
+ :param string: any string
70
+ :param replace: str replace special characters with this
71
+ :return: string with special characters replaced
72
+ """
73
+ return re_special_characters.sub(replace, string)
74
+
75
+
76
+ def round_string_floats(string):
77
+ """
78
+ Shorten string by removing long floats
79
+ :param string: string, e.g. '#810002 scan eta 74.89533603616637 76.49533603616636 0.02 pil3_100k 1 roi2'
80
+ :return: shorter string, e.g. '#810002 scan eta 74.895 76.495 0.02 pil3_100k 1 roi2'
81
+ """
82
+ def subfun(m):
83
+ return str(round(float(m.group()), 3))
84
+ return re_long_floats.sub(subfun, string)
85
+
86
+
87
+ def is_image(shape: tuple[int], min_dim=3):
88
+ """Return True/False if dataset shape is suitable for image data"""
89
+ return len(shape) >= min_dim and (shape[-2] - 1) * (shape[-1] - 1) > 1
90
+
91
+
92
+ def generate_image_roi_slice(start_i: int, stop_i: int, start_j: int, stop_j: int, step_i: int = 1, step_j: int = 1) -> tuple[EllipsisType, slice, slice]:
93
+ """
94
+ Generate indexing slice for region of interest (ROI)
95
+ """
96
+ return Ellipsis, slice(start_i, stop_i, step_i), slice(start_j, stop_j, step_j)
97
+
98
+
99
+ def make_json_serialisable(obj):
100
+ """Convert h5py/numpy objects into JSON-compatible types."""
101
+ if isinstance(obj, bytes):
102
+ return obj.decode()
103
+
104
+ if isinstance(obj, np.ndarray):
105
+ return [make_json_serialisable(v) for v in obj.tolist()]
106
+
107
+ if isinstance(obj, np.generic):
108
+ return obj.item()
109
+
110
+ if isinstance(obj, dict):
111
+ return {k: make_json_serialisable(v) for k, v in obj.items()}
112
+
113
+ if isinstance(obj, (list, tuple)):
114
+ return [make_json_serialisable(v) for v in obj]
115
+
116
+ return obj
117
+
118
+
119
+ def attrs2dict(obj: h5py.Group | h5py.Dataset) -> dict:
120
+ """Extract obj.attrs as a serialisable dict"""
121
+ return {name: make_json_serialisable(item) for name, item in obj.attrs.items()}
122
+
123
+
124
+ def dataset2data(dataset: h5py.Dataset, index: int | slice = (), direct_load=False) -> datetime.datetime | str | np.ndarray:
125
+ """
126
+ Read the data from a h5py Dataset and convert to either datetime, str or squeezed numpy array
127
+
128
+ - numeric size=1 datasets return float
129
+ - numeric size>1 datasets return numpy array
130
+ - string/ byte timestamps (size=1) return datetime.datetime object
131
+ - string/ byte timestamps (size>1) return array of datetime.datetime
132
+ - all other size=1 datasets return string
133
+ - all other size>1 datasets return array of string
134
+
135
+ :param dataset: h5py.Dataset containing data
136
+ :param index: index of array (not used if dataset is string/ bytes type)
137
+ :param direct_load: loads the data directly without conversion if True
138
+ :return datetime.datetime: if data is an isoformat string, returns datetime object
139
+ :return str: if data is another string, returns str with long floats rounded
140
+ :return np.ndarray: if data is another numeric object
141
+ """
142
+ if direct_load:
143
+ return dataset[index]
144
+ if np.issubdtype(dataset, np.number):
145
+ logger.debug(f"Dataset {repr(dataset)} is numeric, return numpy array")
146
+ # note that squeeze returns float if np.float64 is given.
147
+ return np.squeeze(dataset[index]) # numeric np.ndarray
148
+ try:
149
+ # str integers will be cast as timestamps (years), capture as int
150
+ return np.squeeze(dataset[index]).astype(int)
151
+ except ValueError:
152
+ pass
153
+ try:
154
+ # timestamp -> np.datetime64 -> datetime, results in Numpy warnings for timezones, wrong time
155
+ # timestamp = np.squeeze(dataset[index]).astype(np.datetime64).astype(datetime.datetime)
156
+ timestamp = np.squeeze(datetime_converter(dataset))
157
+ logger.debug(f"Dataset {repr(dataset)} is timestamp, return array of datetime objects")
158
+ # single datetime obj vs array of datetime obj
159
+ return timestamp[()] if timestamp.ndim == 0 else timestamp
160
+ except (ValueError, OSError): # OSError arises sometimes for reasons I don't understand (e.g. old I06-1 data)
161
+ try:
162
+ string_dataset = dataset.asstr()[()]
163
+ logger.debug(f"Dataset {repr(dataset)} is string")
164
+ if dataset.ndim == 0:
165
+ return round_string_floats(string_dataset) # bytes or str -> str
166
+ return string_dataset[index] # str array
167
+ except ValueError:
168
+ logger.debug(f"Dataset {repr(dataset)} is an unexpected type")
169
+ return np.squeeze(dataset[index]) # other np.ndarray
170
+
171
+
172
+ def dataset2str(dataset: h5py.Dataset, index: int | slice | tuple[slice, ...] = (), units: bool = False) -> str:
173
+ """
174
+ Read the data from a h5py Dataset and convert to a representative string
175
+
176
+ - Strings are given with quotes
177
+ - long floats are shorted by attribute 'decimals'
178
+ - numeric arrays are summarised as "dtype (shape)"
179
+ - string arrays are summarised as "['str0', ...]
180
+ - trailing floats within strings are shortened
181
+ - if units=True and units available as attribute, the unit will be appended
182
+
183
+ :param dataset: h5py.Dataset containing data
184
+ :param index: index of array (not used if dataset is string/ bytes type)
185
+ :param units: if True and attribute 'units' available, append this to the result
186
+ :return str: string representation of data
187
+ """
188
+ if np.issubdtype(dataset, np.number):
189
+ logger.debug(f"Dataset {repr(dataset)} is numeric")
190
+ if dataset.size > 1:
191
+ return f"{dataset.dtype} {dataset.shape}"
192
+ value = np.squeeze(dataset[index]) # size 1 numeric np.ndarray
193
+ if 'decimals' in dataset.attrs:
194
+ decimals = int(dataset.attrs['decimals'])
195
+ if abs(value) > 1:
196
+ value = value.round(decimals)
197
+ value = np.array2string(value, precision=decimals, separator=', ', floatmode='maxprec')
198
+ if units and 'units' in dataset.attrs:
199
+ value = f"{value} {arg.decode() if isinstance((arg := dataset.attrs['units']), bytes) else arg}"
200
+ return str(value)
201
+ try:
202
+ # timestamp -> datetime64 -> datetime
203
+ # timestamp = np.squeeze(dataset[index]).astype(np.datetime64).astype(datetime.datetime)
204
+ timestamp = np.squeeze(datetime_converter(dataset))
205
+ logger.debug(f"Dataset {repr(dataset)} is timestamp")
206
+ # single datetime obj vs array of datetime obj
207
+ return f"'{timestamp[()]}'" if timestamp.ndim == 0 else f"['{timestamp[0]}', ...({len(timestamp)})]"
208
+ except (ValueError, OSError): # OSError arises sometimes for reasons I don't understand (e.g. old I06-1 data)
209
+ try:
210
+ string_dataset = dataset.asstr()[()]
211
+ logger.debug(f"Dataset {repr(dataset)} is string")
212
+ if dataset.ndim == 0:
213
+ return f"'{round_string_floats(string_dataset)}'" # bytes or str -> str
214
+ return f"['{string_dataset[0]}', ...({len(string_dataset)})]" # str array
215
+ except ValueError:
216
+ logger.debug(f"Dataset {repr(dataset)} is an unexpected type")
217
+ return str(np.squeeze(dataset[index])) # other np.ndarray
218
+
219
+
220
+ def dataset_attribute(dataset: h5py.Dataset | Dataset, attribute: str) -> str:
221
+ """
222
+ Return attribute of dataset
223
+ """
224
+ value = dataset.attrs.get(attribute, '')
225
+ if isinstance(value, bytes):
226
+ return value.decode()
227
+ return value