blissdata 2.3.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {blissdata-2.3.0/src/blissdata.egg-info → blissdata-2.4.0}/PKG-INFO +4 -4
- {blissdata-2.3.0 → blissdata-2.4.0}/pyproject.toml +4 -4
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/hdf5.py +12 -1
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5map/__init__.py +1 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5map/data_items.py +74 -9
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/lima/image_utils.py +2 -2
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/memory_tracker.py +11 -4
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/store.py +2 -2
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/schemas/scan_info_display.py +0 -1
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/test_files.py +2 -2
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/conftest.py +0 -1
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_dynamic_files.py +89 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_redis_files.py +6 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5map/test_data_items.py +91 -1
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/lima/test_image_utils.py +2 -2
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_memory_tracker.py +11 -5
- {blissdata-2.3.0 → blissdata-2.4.0/src/blissdata.egg-info}/PKG-INFO +4 -4
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/requires.txt +3 -3
- {blissdata-2.3.0 → blissdata-2.4.0}/LICENSE +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/README.md +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/setup.cfg +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/_base.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/_utils.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/config.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/data.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/files.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/deprecation.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/exceptions.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/abstract.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/dynamic_hdf5.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/file_arguments.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/h5_scan_map.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/redis_hdf5.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/static_hdf5.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/bliss.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/hdf5_retry.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/lima.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/types.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5map/hdf5_items.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/lima/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/lima/client.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/py.typed +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/encoding/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/encoding/json.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/encoding/numeric.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/event_stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/exceptions.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/identities.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/scan.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/scan.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/schemas/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/schemas/scan_info.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/base.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/default.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/encoding/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/encoding/json.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/encoding/numeric.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/event_stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/hdf5_fallback/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/hdf5_fallback/stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/lima/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/lima/stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/scan_sequence/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/scan_sequence/stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/sink.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/test_config.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/test_data.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/scanner.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_h5api_lima_utils.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_hdf5_handlers.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_static_files.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5map/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5map/test_hdf5_items.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/lima/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/conftest.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_cursor.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_scan.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_search.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/utils.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/encoding/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/encoding/test_json.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/encoding/test_numeric.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/__init__.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/test_file_backed_stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/test_plugin_base.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/test_event_stream.py +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/SOURCES.txt +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/dependency_links.txt +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/entry_points.txt +0 -0
- {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: blissdata
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: Bliss data streaming client
|
|
5
5
|
Maintainer: BCU (ESRF)
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
@@ -19,9 +19,9 @@ Requires-Dist: h5py>=2.9
|
|
|
19
19
|
Requires-Dist: msgpack
|
|
20
20
|
Requires-Dist: numpy>=1.20
|
|
21
21
|
Requires-Dist: numpydantic<1.6.10
|
|
22
|
-
Requires-Dist: pydantic>=2.
|
|
22
|
+
Requires-Dist: pydantic>=2.9
|
|
23
23
|
Requires-Dist: redis>=4.6.0
|
|
24
|
-
Requires-Dist: redis-om
|
|
24
|
+
Requires-Dist: redis-om~=1.0
|
|
25
25
|
Requires-Dist: silx>=1.1.0
|
|
26
26
|
Provides-Extra: tango
|
|
27
27
|
Requires-Dist: pytango>=9.3.1; extra == "tango"
|
|
@@ -36,7 +36,7 @@ Requires-Dist: pytest-timeout; extra == "test"
|
|
|
36
36
|
Requires-Dist: packaging; extra == "test"
|
|
37
37
|
Provides-Extra: dev
|
|
38
38
|
Requires-Dist: blissdata[test]; extra == "dev"
|
|
39
|
-
Requires-Dist: black
|
|
39
|
+
Requires-Dist: black; extra == "dev"
|
|
40
40
|
Requires-Dist: flake8>=4; extra == "dev"
|
|
41
41
|
Requires-Dist: flake8-pep585; extra == "dev"
|
|
42
42
|
Requires-Dist: mkdocs; extra == "dev"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "blissdata"
|
|
3
|
-
version = "2.
|
|
3
|
+
version = "2.4.0"
|
|
4
4
|
description = "Bliss data streaming client"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "LGPL-3.0-or-later"
|
|
@@ -19,9 +19,9 @@ dependencies = [
|
|
|
19
19
|
"msgpack",
|
|
20
20
|
"numpy>=1.20",
|
|
21
21
|
"numpydantic<1.6.10",
|
|
22
|
-
"pydantic>=2.
|
|
22
|
+
"pydantic>=2.9",
|
|
23
23
|
"redis>=4.6.0",
|
|
24
|
-
"redis-om
|
|
24
|
+
"redis-om~=1.0",
|
|
25
25
|
"silx>=1.1.0",
|
|
26
26
|
]
|
|
27
27
|
|
|
@@ -43,7 +43,7 @@ test = [
|
|
|
43
43
|
]
|
|
44
44
|
dev = [
|
|
45
45
|
"blissdata[test]",
|
|
46
|
-
"black
|
|
46
|
+
"black",
|
|
47
47
|
"flake8>=4",
|
|
48
48
|
"flake8-pep585",
|
|
49
49
|
"mkdocs",
|
|
@@ -251,7 +251,7 @@ class DynamicHDF5Handler:
|
|
|
251
251
|
try:
|
|
252
252
|
return item.attrs[key] # try one more time
|
|
253
253
|
except KeyError as e:
|
|
254
|
-
if not
|
|
254
|
+
if not _attribute_does_not_exist(e):
|
|
255
255
|
raise
|
|
256
256
|
# attribute will never be created
|
|
257
257
|
raise RetryTimeoutError(str(e)) from e
|
|
@@ -366,3 +366,14 @@ def _key_does_not_exist(exc: KeyError) -> bool:
|
|
|
366
366
|
|
|
367
367
|
"""
|
|
368
368
|
return "doesn't exist" in str(exc)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _attribute_does_not_exist(exc: KeyError) -> bool:
|
|
372
|
+
"""
|
|
373
|
+
A real KeyError (meaning that the attribute does not exist) gives the following error message:
|
|
374
|
+
|
|
375
|
+
.. code
|
|
376
|
+
|
|
377
|
+
KeyError: "Unable to synchronously open attribute (can't locate attribute: 'myattr')"
|
|
378
|
+
"""
|
|
379
|
+
return "can't locate attribute" in str(exc)
|
|
@@ -6,6 +6,7 @@ from pydantic import (
|
|
|
6
6
|
Base64Bytes,
|
|
7
7
|
Field,
|
|
8
8
|
field_validator,
|
|
9
|
+
model_validator,
|
|
9
10
|
model_serializer,
|
|
10
11
|
Discriminator,
|
|
11
12
|
Tag,
|
|
@@ -64,7 +65,13 @@ class InlineRaw(RootModel):
|
|
|
64
65
|
def decode(self):
|
|
65
66
|
value = self.model_dump()
|
|
66
67
|
if value.dtype.kind in "UT":
|
|
67
|
-
|
|
68
|
+
# Make strings similar to those read back from h5py:
|
|
69
|
+
# - Convert to "object" dtype
|
|
70
|
+
# - Tell h5py that "object" is variable length string using
|
|
71
|
+
# metadata, so writing doesn't fail even in a compound dataset.
|
|
72
|
+
return np.vectorize(
|
|
73
|
+
lambda x: x.encode(), otypes=[np.dtype("O", metadata={"vlen": str})]
|
|
74
|
+
)(value)
|
|
68
75
|
else:
|
|
69
76
|
return value
|
|
70
77
|
|
|
@@ -155,14 +162,6 @@ class InlineBase64(BaseModel):
|
|
|
155
162
|
return value
|
|
156
163
|
|
|
157
164
|
|
|
158
|
-
class Stream(BaseModel):
|
|
159
|
-
type: Literal["stream"] = "stream"
|
|
160
|
-
stream: str
|
|
161
|
-
|
|
162
|
-
def decode(self):
|
|
163
|
-
raise TypeError("Stream model is just a reference, can't use decode()")
|
|
164
|
-
|
|
165
|
-
|
|
166
165
|
def get_discriminator_value(v: Any) -> str:
|
|
167
166
|
if isinstance(v, dict):
|
|
168
167
|
return v.get("type")
|
|
@@ -171,12 +170,78 @@ def get_discriminator_value(v: Any) -> str:
|
|
|
171
170
|
return "raw"
|
|
172
171
|
|
|
173
172
|
|
|
173
|
+
class Compound(BaseModel):
|
|
174
|
+
"""Composite model to combine multiple DataItems as a structured array.
|
|
175
|
+
Unlike numpy structured arrays where data is a list of heterogeneous tuples,
|
|
176
|
+
here all elements of one dtype are stored together. This way existing
|
|
177
|
+
DataItem models can be reused as columns.
|
|
178
|
+
|
|
179
|
+
Validation ensures:
|
|
180
|
+
- Column names uniqueness
|
|
181
|
+
- Shape consistency between columns
|
|
182
|
+
"""
|
|
183
|
+
|
|
184
|
+
type: Literal["compound"] = "compound"
|
|
185
|
+
columns: list[
|
|
186
|
+
tuple[
|
|
187
|
+
str,
|
|
188
|
+
Annotated[
|
|
189
|
+
(
|
|
190
|
+
Annotated[Inline, Tag("inline")]
|
|
191
|
+
| Annotated[InlineBase64, Tag("inline_b64")]
|
|
192
|
+
| Annotated[InlineRaw, Tag("raw")]
|
|
193
|
+
| Annotated["Compound", Tag("compound")]
|
|
194
|
+
# Note: Stream can't be part of a compound object
|
|
195
|
+
),
|
|
196
|
+
Discriminator(get_discriminator_value),
|
|
197
|
+
],
|
|
198
|
+
]
|
|
199
|
+
] = Field(min_length=1)
|
|
200
|
+
# Reusing DataItemType simplifies this model as it does not redefine
|
|
201
|
+
# handling of dtype, shape or base64 version of the values.
|
|
202
|
+
# Therefore the representation is the transposed of numpy one, which can be
|
|
203
|
+
# harder to read.
|
|
204
|
+
|
|
205
|
+
def decode(self):
|
|
206
|
+
decoded_columns = [(name, value.decode()) for name, value in self.columns]
|
|
207
|
+
rec = np.rec.fromarrays(
|
|
208
|
+
[value for _, value in decoded_columns],
|
|
209
|
+
names=[name for name, _ in decoded_columns],
|
|
210
|
+
)
|
|
211
|
+
return rec.view(np.ndarray)
|
|
212
|
+
|
|
213
|
+
@model_validator(mode="after")
|
|
214
|
+
@classmethod
|
|
215
|
+
def verify_names_and_shapes(cls, value):
|
|
216
|
+
"""Ensure uniqueness of names and shape equality between columns"""
|
|
217
|
+
if len(value.columns) != len({name for name, _ in value.columns}):
|
|
218
|
+
raise ValueError("found columns with same name")
|
|
219
|
+
|
|
220
|
+
shape_iterator = iter((col.decode().shape for _, col in value.columns))
|
|
221
|
+
try:
|
|
222
|
+
first_shape = next(shape_iterator)
|
|
223
|
+
except StopIteration:
|
|
224
|
+
return value
|
|
225
|
+
if not all(first_shape == shape for shape in shape_iterator):
|
|
226
|
+
raise ValueError("columns have different shapes")
|
|
227
|
+
return value
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class Stream(BaseModel):
|
|
231
|
+
type: Literal["stream"] = "stream"
|
|
232
|
+
stream: str
|
|
233
|
+
|
|
234
|
+
def decode(self):
|
|
235
|
+
raise TypeError("Stream model is just a reference, can't use decode()")
|
|
236
|
+
|
|
237
|
+
|
|
174
238
|
DataItemType = Annotated[
|
|
175
239
|
(
|
|
176
240
|
Annotated[Inline, Tag("inline")]
|
|
177
241
|
| Annotated[InlineBase64, Tag("inline_b64")]
|
|
178
242
|
| Annotated[Stream, Tag("stream")]
|
|
179
243
|
| Annotated[InlineRaw, Tag("raw")]
|
|
244
|
+
| Annotated[Compound, Tag("compound")]
|
|
180
245
|
),
|
|
181
246
|
Discriminator(get_discriminator_value),
|
|
182
247
|
]
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
# Copyright (c) Beamline Control Unit, ESRF
|
|
6
6
|
# Distributed under the GNU LGPLv3. See LICENSE for more info.
|
|
7
7
|
|
|
8
|
-
"""Utility functions to read Lima images from file or device server.
|
|
9
|
-
|
|
8
|
+
"""Utility functions to read Lima images from file or device server."""
|
|
9
|
+
|
|
10
10
|
import struct
|
|
11
11
|
import numpy
|
|
12
12
|
import typing
|
|
@@ -16,7 +16,6 @@ from dataclasses import dataclass
|
|
|
16
16
|
from blissdata import DataStore, Scan, ScanState
|
|
17
17
|
from blissdata.exceptions import NoScanAvailable, ScanNotFoundError, ScanLoadError
|
|
18
18
|
|
|
19
|
-
|
|
20
19
|
memory_tracking_stream = "_MEMORY_TRACKING_"
|
|
21
20
|
SEAL_ID = ("-".join([str(2**64 - 1)] * 2)).encode()
|
|
22
21
|
|
|
@@ -35,6 +34,7 @@ class NoFreeableMemoryError(Exception):
|
|
|
35
34
|
class TrackedScan:
|
|
36
35
|
scan: Scan
|
|
37
36
|
last_activity: int
|
|
37
|
+
last_streams_ids: dict[str, int]
|
|
38
38
|
|
|
39
39
|
def __eq__(self, other):
|
|
40
40
|
return self.scan.key == other.scan.key
|
|
@@ -93,7 +93,7 @@ class MemoryTracker:
|
|
|
93
93
|
scan._model.state_stream.key(), count=1
|
|
94
94
|
)
|
|
95
95
|
last_activity = int(raw[0][0].split(b"-")[0]) / 1000
|
|
96
|
-
self._tracked_scans.add(TrackedScan(scan, last_activity))
|
|
96
|
+
self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
|
|
97
97
|
except ScanNotFoundError:
|
|
98
98
|
# scan already deleted from Redis by user, skip it
|
|
99
99
|
pass
|
|
@@ -113,7 +113,7 @@ class MemoryTracker:
|
|
|
113
113
|
try:
|
|
114
114
|
scan = self._data_store.load_scan(scan_key)
|
|
115
115
|
last_activity = int(since.split("-")[0]) / 1000
|
|
116
|
-
self._tracked_scans.add(TrackedScan(scan, last_activity))
|
|
116
|
+
self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
|
|
117
117
|
self._since = since
|
|
118
118
|
logging.info(f"Start tracking {scan.key}")
|
|
119
119
|
except ScanNotFoundError:
|
|
@@ -135,7 +135,14 @@ class MemoryTracker:
|
|
|
135
135
|
|
|
136
136
|
if track.scan.state >= ScanState.PREPARED:
|
|
137
137
|
streams_ids = self._data_streams_ids(track.scan)
|
|
138
|
-
|
|
138
|
+
stream_activity = {
|
|
139
|
+
key: idx
|
|
140
|
+
for key, idx in streams_ids.items()
|
|
141
|
+
if key not in track.last_streams_ids
|
|
142
|
+
or track.last_streams_ids[key] != idx
|
|
143
|
+
}
|
|
144
|
+
if stream_activity:
|
|
145
|
+
track.last_streams_ids = streams_ids
|
|
139
146
|
track.last_activity, _ = self._data_store._redis.time()
|
|
140
147
|
all_data_streams_ids.update(streams_ids)
|
|
141
148
|
|
|
@@ -43,7 +43,7 @@ class DataStore:
|
|
|
43
43
|
close_func = type(redis_pool).disconnect
|
|
44
44
|
weakref.finalize(self, close_func, redis_pool)
|
|
45
45
|
|
|
46
|
-
class StreamModel(HashModel):
|
|
46
|
+
class StreamModel(HashModel, index=True):
|
|
47
47
|
"""Actual stream model to be embedded into ScanModel, but not exposed to user directly.
|
|
48
48
|
They are wrapped into Stream objects instead."""
|
|
49
49
|
|
|
@@ -60,7 +60,7 @@ class DataStore:
|
|
|
60
60
|
encoding: dict | str | None = None
|
|
61
61
|
info: dict | str | None = None
|
|
62
62
|
|
|
63
|
-
class ScanModel(JsonModel):
|
|
63
|
+
class ScanModel(JsonModel, index=True):
|
|
64
64
|
class Meta:
|
|
65
65
|
global_key_prefix = identity_model_cls._meta.global_key_prefix
|
|
66
66
|
model_key_prefix = "scan"
|
|
@@ -23,7 +23,7 @@ def test_param_constructor(mocker):
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
def test_get_file(mocker):
|
|
26
|
-
RESPONSE1 = b"\x34\x00\x00\x00\
|
|
26
|
+
RESPONSE1 = b"\x34\x00\x00\x00\x0e\x00\x00\x001|MY FILE DATA"
|
|
27
27
|
mock_socket = mocker.patch("socket.socket")
|
|
28
28
|
mock_socket.return_value.recv.side_effect = [RESPONSE1]
|
|
29
29
|
service = BeaconFiles("foo", 200)
|
|
@@ -35,7 +35,7 @@ def test_get_file(mocker):
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
def test_get_tree(mocker):
|
|
38
|
-
RESPONSE1 = b"""\x58\x00\x00\x00\
|
|
38
|
+
RESPONSE1 = b"""\x58\x00\x00\x00\x2c\x00\x00\x001|{"afile":null,"adir":{"anotherfile":null}}"""
|
|
39
39
|
mock_socket = mocker.patch("socket.socket")
|
|
40
40
|
mock_socket.return_value.recv.side_effect = [RESPONSE1]
|
|
41
41
|
service = BeaconFiles("foo", 200)
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
import h5py
|
|
2
5
|
import numpy
|
|
3
6
|
import pytest
|
|
7
|
+
|
|
4
8
|
from blissdata.h5api import dynamic_hdf5
|
|
5
9
|
from . import scanner
|
|
6
10
|
|
|
@@ -325,3 +329,88 @@ def read_scan_data(nxroot, scan_number, counters, queue):
|
|
|
325
329
|
numpy.testing.assert_array_equal(data, expected)
|
|
326
330
|
scanner.log_point(scan_number, scan_index, queue, write=False)
|
|
327
331
|
print(f"Finished reading scan {scan_number}")
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
@pytest.fixture
|
|
335
|
+
def h5file_finished_scan(tmp_path):
|
|
336
|
+
path = tmp_path / "test.h5"
|
|
337
|
+
with h5py.File(path, "w") as fh:
|
|
338
|
+
fh["/1.1/start_time"] = "2026-01-09T14:49:39.364371+01:00"
|
|
339
|
+
fh["/1.1/end_time"] = "2026-01-09T14:49:39.364371+01:00"
|
|
340
|
+
return str(path)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@pytest.fixture
|
|
344
|
+
def h5file_running_scan(tmp_path):
|
|
345
|
+
path = tmp_path / "test.h5"
|
|
346
|
+
with h5py.File(path, "w") as fh:
|
|
347
|
+
fh["/1.1/start_time"] = "2026-01-09T14:49:39.364371+01:00"
|
|
348
|
+
return str(path)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def test_non_existing_h5item_in_finished_scan(h5file_finished_scan):
|
|
352
|
+
_assert_non_existing_h5item(h5file_finished_scan, finished=True)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def test_non_existing_h5item_in_running_scan(h5file_running_scan):
|
|
356
|
+
_assert_non_existing_h5item(h5file_running_scan, finished=False)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def test_non_existing_h5attr_in_finished_scan(h5file_finished_scan):
|
|
360
|
+
_assert_non_existing_h5attr(h5file_finished_scan, finished=True)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def test_non_existing_h5attr_in_running_scan(h5file_running_scan):
|
|
364
|
+
_assert_non_existing_h5attr(h5file_running_scan, finished=False)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _assert_non_existing_h5item(h5file, finished):
|
|
368
|
+
retry_timeout = 2
|
|
369
|
+
max_immediate_time = 0.5
|
|
370
|
+
|
|
371
|
+
with dynamic_hdf5.File(h5file, retry_timeout=retry_timeout) as fh:
|
|
372
|
+
scan = fh["/1.1"]
|
|
373
|
+
|
|
374
|
+
t0 = time.time()
|
|
375
|
+
with pytest.raises(KeyError):
|
|
376
|
+
scan["instrument"]
|
|
377
|
+
elapsed = time.time() - t0
|
|
378
|
+
if finished:
|
|
379
|
+
assert elapsed < max_immediate_time
|
|
380
|
+
else:
|
|
381
|
+
assert elapsed >= retry_timeout
|
|
382
|
+
|
|
383
|
+
t0 = time.time()
|
|
384
|
+
value = scan.get("instrument")
|
|
385
|
+
elapsed = time.time() - t0
|
|
386
|
+
assert value is None
|
|
387
|
+
if finished:
|
|
388
|
+
assert elapsed < max_immediate_time
|
|
389
|
+
else:
|
|
390
|
+
assert elapsed >= retry_timeout
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _assert_non_existing_h5attr(h5file, finished):
|
|
394
|
+
retry_timeout = 2
|
|
395
|
+
max_immediate_time = 0.5
|
|
396
|
+
|
|
397
|
+
with dynamic_hdf5.File(h5file, retry_timeout=retry_timeout) as fh:
|
|
398
|
+
scan = fh["/1.1"]
|
|
399
|
+
|
|
400
|
+
t0 = time.time()
|
|
401
|
+
with pytest.raises(KeyError):
|
|
402
|
+
scan.attrs["myattr"]
|
|
403
|
+
elapsed = time.time() - t0
|
|
404
|
+
if finished:
|
|
405
|
+
assert elapsed < max_immediate_time
|
|
406
|
+
else:
|
|
407
|
+
assert elapsed >= retry_timeout
|
|
408
|
+
|
|
409
|
+
t0 = time.time()
|
|
410
|
+
value = scan.attrs.get("myattr")
|
|
411
|
+
elapsed = time.time() - t0
|
|
412
|
+
assert value is None
|
|
413
|
+
if finished:
|
|
414
|
+
assert elapsed < max_immediate_time
|
|
415
|
+
else:
|
|
416
|
+
assert elapsed >= retry_timeout
|
|
@@ -307,6 +307,12 @@ def test_external_link(data_store, tmpdir):
|
|
|
307
307
|
h5m.InlineRaw(["a", "b", "c"]),
|
|
308
308
|
h5m.Inline(value=1.23, dtype="float16"),
|
|
309
309
|
h5m.Inline(value=[1.23, "4.56"], dtype="float32"),
|
|
310
|
+
h5m.Compound(
|
|
311
|
+
columns=[
|
|
312
|
+
("name", ["chan1", "chan2"]),
|
|
313
|
+
("current", h5m.Inline(value=[1.23, "4.56"], dtype="float32")),
|
|
314
|
+
]
|
|
315
|
+
),
|
|
310
316
|
],
|
|
311
317
|
)
|
|
312
318
|
def test_static_dataset(data_store, tmpdir, value):
|
|
@@ -2,7 +2,7 @@ import pytest
|
|
|
2
2
|
import pydantic
|
|
3
3
|
import base64
|
|
4
4
|
import numpy as np
|
|
5
|
-
from blissdata.h5map import InlineRaw, Inline, InlineBase64
|
|
5
|
+
from blissdata.h5map import InlineRaw, Inline, InlineBase64, Compound, Stream
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
@pytest.mark.parametrize(
|
|
@@ -211,6 +211,96 @@ def test_inline_load(json_input, expected_value):
|
|
|
211
211
|
assert output.dtype == expected_value.dtype
|
|
212
212
|
|
|
213
213
|
|
|
214
|
+
@pytest.mark.parametrize(
|
|
215
|
+
"input_value, expected_json",
|
|
216
|
+
[
|
|
217
|
+
pytest.param(
|
|
218
|
+
[("a", 123)],
|
|
219
|
+
'{"type":"compound","columns":[["a",123]]}',
|
|
220
|
+
id="single column",
|
|
221
|
+
),
|
|
222
|
+
pytest.param(
|
|
223
|
+
[("a", 1), ("b", 2)],
|
|
224
|
+
'{"type":"compound","columns":[["a",1],["b",2]]}',
|
|
225
|
+
id="scalars",
|
|
226
|
+
),
|
|
227
|
+
pytest.param(
|
|
228
|
+
[("a", [1, 2]), ("b", [3.0, 4.0])],
|
|
229
|
+
'{"type":"compound","columns":[["a",[1,2]],["b",[3.0,4.0]]]}',
|
|
230
|
+
id="arrays",
|
|
231
|
+
),
|
|
232
|
+
pytest.param(
|
|
233
|
+
[("a", [[1, 2], [3, 4]]), ("b", [[5.0, 6.0], [7.0, 8.0]])],
|
|
234
|
+
'{"type":"compound","columns":[["a",[[1,2],[3,4]]],["b",[[5.0,6.0],[7.0,8.0]]]]}',
|
|
235
|
+
id="ndarrays",
|
|
236
|
+
),
|
|
237
|
+
pytest.param(
|
|
238
|
+
[("name", ["alice", "bob"]), ("id", [123, 456])],
|
|
239
|
+
'{"type":"compound","columns":[["name",["alice","bob"]],["id",[123,456]]]}',
|
|
240
|
+
id="strings",
|
|
241
|
+
),
|
|
242
|
+
pytest.param(
|
|
243
|
+
[
|
|
244
|
+
("a", Inline(value=[1.2, 2.3], dtype="float16")),
|
|
245
|
+
("b", Inline(value=[12, 23], dtype="uint32")),
|
|
246
|
+
],
|
|
247
|
+
'{"type":"compound","columns":[["a",{"type":"inline","dtype":"float16","value":[1.2,2.3]}],["b",{"type":"inline","dtype":"uint32","value":[12,23]}]]}',
|
|
248
|
+
id="explicit types",
|
|
249
|
+
),
|
|
250
|
+
pytest.param(
|
|
251
|
+
[
|
|
252
|
+
["id", [1, 2, 3]],
|
|
253
|
+
[
|
|
254
|
+
"person",
|
|
255
|
+
Compound(
|
|
256
|
+
columns=[
|
|
257
|
+
["name", ["Alice", "Bob", "Charlie"]],
|
|
258
|
+
[
|
|
259
|
+
"email",
|
|
260
|
+
[
|
|
261
|
+
"alice@example.com",
|
|
262
|
+
"bob@example.com",
|
|
263
|
+
"charlie@example.com",
|
|
264
|
+
],
|
|
265
|
+
],
|
|
266
|
+
]
|
|
267
|
+
),
|
|
268
|
+
],
|
|
269
|
+
],
|
|
270
|
+
'{"type":"compound","columns":[["id",[1,2,3]],["person",{"type":"compound","columns":[["name",["Alice","Bob","Charlie"]],["email",["alice@example.com","bob@example.com","charlie@example.com"]]]}]]}',
|
|
271
|
+
id="nested",
|
|
272
|
+
),
|
|
273
|
+
],
|
|
274
|
+
)
|
|
275
|
+
def test_compound(input_value, expected_json):
|
|
276
|
+
model = Compound(columns=input_value)
|
|
277
|
+
assert model.model_dump_json() == expected_json
|
|
278
|
+
_ = model.decode()
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@pytest.mark.parametrize(
|
|
282
|
+
"input_value",
|
|
283
|
+
[
|
|
284
|
+
pytest.param([], id="empty"),
|
|
285
|
+
pytest.param(
|
|
286
|
+
[("aaa", [1, 2]), ("bbb", ["hello", "bye"]), ("ccc", [1.23])],
|
|
287
|
+
id="misaligned",
|
|
288
|
+
),
|
|
289
|
+
pytest.param(
|
|
290
|
+
[("stream", Stream(stream="stream_name")), ("integer", 42)],
|
|
291
|
+
id="stream",
|
|
292
|
+
),
|
|
293
|
+
pytest.param(
|
|
294
|
+
[("twin", 1), ("twin", 2)],
|
|
295
|
+
id="duplicate name",
|
|
296
|
+
),
|
|
297
|
+
],
|
|
298
|
+
)
|
|
299
|
+
def test_compound_forbidden_value(input_value):
|
|
300
|
+
with pytest.raises(pydantic.ValidationError):
|
|
301
|
+
_ = Compound(columns=input_value)
|
|
302
|
+
|
|
303
|
+
|
|
214
304
|
@pytest.mark.parametrize(
|
|
215
305
|
"json_input, expected_value",
|
|
216
306
|
[
|
|
@@ -63,13 +63,13 @@ RAW_YUV422PACKED_VIDEO = (
|
|
|
63
63
|
RAW_RGB24_VIDEO = (
|
|
64
64
|
b"VDEO\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
|
|
65
65
|
\x00\x02\x00\x00\x00\x02\x00\x00\x00 \x00\x00\x00\x00"
|
|
66
|
-
+ b"\
|
|
66
|
+
+ b"\xff\x00\x00\x00\xff\x00\x00\x00\xff\x00\x00\x00"
|
|
67
67
|
)
|
|
68
68
|
|
|
69
69
|
RAW_RGB32_VIDEO = (
|
|
70
70
|
b"VDEO\x00\x01\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\
|
|
71
71
|
\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00 \x00\x00\x00\x00"
|
|
72
|
-
+ b"\
|
|
72
|
+
+ b"\xff\x00\x00\x00\x00\xff\x00\x00\x00\x00\xff\x00\x00\x00\x00\xff"
|
|
73
73
|
)
|
|
74
74
|
|
|
75
75
|
RAW_BAYER_RG16 = (
|
|
@@ -224,29 +224,35 @@ def test_delete_inactive_scan(redis_url, data_store, dummy_id):
|
|
|
224
224
|
active_scan = data_store.create_scan(dummy_id)
|
|
225
225
|
|
|
226
226
|
stream_definition = Stream.make_definition("mystream", "int")
|
|
227
|
-
|
|
227
|
+
stream_ina = inactive_scan.create_stream(stream_definition)
|
|
228
228
|
|
|
229
|
+
stream_definition = Stream.make_definition("mystream", "int")
|
|
230
|
+
stream_act = active_scan.create_stream(stream_definition)
|
|
231
|
+
|
|
232
|
+
inactive_scan.prepare()
|
|
233
|
+
inactive_scan.start()
|
|
229
234
|
active_scan.prepare()
|
|
230
235
|
active_scan.start()
|
|
231
236
|
|
|
232
|
-
# wait for tracker to find the
|
|
237
|
+
# wait for tracker to find the scans
|
|
233
238
|
nb_scans_to_track = 2
|
|
234
239
|
while nb_scans_to_track:
|
|
235
|
-
|
|
240
|
+
stream_ina.send(1)
|
|
241
|
+
stream_act.send(1)
|
|
236
242
|
line = proc.stdout.readline()
|
|
237
243
|
print(line, end="")
|
|
238
244
|
if "Start tracking" in line:
|
|
239
245
|
nb_scans_to_track -= 1
|
|
240
246
|
|
|
241
247
|
while True:
|
|
242
|
-
|
|
248
|
+
stream_act.send(2)
|
|
243
249
|
line = proc.stdout.readline()
|
|
244
250
|
print(line, end="")
|
|
245
251
|
if f"{inactive_scan.key} was inactive" in line:
|
|
246
252
|
break
|
|
247
253
|
|
|
248
254
|
for _ in range(4):
|
|
249
|
-
|
|
255
|
+
stream_act.send(3)
|
|
250
256
|
line = proc.stdout.readline()
|
|
251
257
|
print(line, end="")
|
|
252
258
|
assert f"{active_scan.key} was inactive" not in line
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: blissdata
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: Bliss data streaming client
|
|
5
5
|
Maintainer: BCU (ESRF)
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
@@ -19,9 +19,9 @@ Requires-Dist: h5py>=2.9
|
|
|
19
19
|
Requires-Dist: msgpack
|
|
20
20
|
Requires-Dist: numpy>=1.20
|
|
21
21
|
Requires-Dist: numpydantic<1.6.10
|
|
22
|
-
Requires-Dist: pydantic>=2.
|
|
22
|
+
Requires-Dist: pydantic>=2.9
|
|
23
23
|
Requires-Dist: redis>=4.6.0
|
|
24
|
-
Requires-Dist: redis-om
|
|
24
|
+
Requires-Dist: redis-om~=1.0
|
|
25
25
|
Requires-Dist: silx>=1.1.0
|
|
26
26
|
Provides-Extra: tango
|
|
27
27
|
Requires-Dist: pytango>=9.3.1; extra == "tango"
|
|
@@ -36,7 +36,7 @@ Requires-Dist: pytest-timeout; extra == "test"
|
|
|
36
36
|
Requires-Dist: packaging; extra == "test"
|
|
37
37
|
Provides-Extra: dev
|
|
38
38
|
Requires-Dist: blissdata[test]; extra == "dev"
|
|
39
|
-
Requires-Dist: black
|
|
39
|
+
Requires-Dist: black; extra == "dev"
|
|
40
40
|
Requires-Dist: flake8>=4; extra == "dev"
|
|
41
41
|
Requires-Dist: flake8-pep585; extra == "dev"
|
|
42
42
|
Requires-Dist: mkdocs; extra == "dev"
|
|
@@ -4,9 +4,9 @@ h5py>=2.9
|
|
|
4
4
|
msgpack
|
|
5
5
|
numpy>=1.20
|
|
6
6
|
numpydantic<1.6.10
|
|
7
|
-
pydantic>=2.
|
|
7
|
+
pydantic>=2.9
|
|
8
8
|
redis>=4.6.0
|
|
9
|
-
redis-om
|
|
9
|
+
redis-om~=1.0
|
|
10
10
|
silx>=1.1.0
|
|
11
11
|
|
|
12
12
|
[beacon]
|
|
@@ -14,7 +14,7 @@ ruamel.yaml
|
|
|
14
14
|
|
|
15
15
|
[dev]
|
|
16
16
|
blissdata[test]
|
|
17
|
-
black
|
|
17
|
+
black
|
|
18
18
|
flake8>=4
|
|
19
19
|
flake8-pep585
|
|
20
20
|
mkdocs
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/test_file_backed_stream.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|