blissdata 2.3.0__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. {blissdata-2.3.0/src/blissdata.egg-info → blissdata-2.4.0}/PKG-INFO +4 -4
  2. {blissdata-2.3.0 → blissdata-2.4.0}/pyproject.toml +4 -4
  3. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/hdf5.py +12 -1
  4. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5map/__init__.py +1 -0
  5. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5map/data_items.py +74 -9
  6. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/lima/image_utils.py +2 -2
  7. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/memory_tracker.py +11 -4
  8. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/store.py +2 -2
  9. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/schemas/scan_info_display.py +0 -1
  10. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/test_files.py +2 -2
  11. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/conftest.py +0 -1
  12. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_dynamic_files.py +89 -0
  13. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_redis_files.py +6 -0
  14. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5map/test_data_items.py +91 -1
  15. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/lima/test_image_utils.py +2 -2
  16. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_memory_tracker.py +11 -5
  17. {blissdata-2.3.0 → blissdata-2.4.0/src/blissdata.egg-info}/PKG-INFO +4 -4
  18. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/requires.txt +3 -3
  19. {blissdata-2.3.0 → blissdata-2.4.0}/LICENSE +0 -0
  20. {blissdata-2.3.0 → blissdata-2.4.0}/README.md +0 -0
  21. {blissdata-2.3.0 → blissdata-2.4.0}/setup.cfg +0 -0
  22. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/__init__.py +0 -0
  23. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/__init__.py +0 -0
  24. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/_base.py +0 -0
  25. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/_utils.py +0 -0
  26. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/config.py +0 -0
  27. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/data.py +0 -0
  28. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/beacon/files.py +0 -0
  29. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/deprecation.py +0 -0
  30. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/exceptions.py +0 -0
  31. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/__init__.py +0 -0
  32. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/abstract.py +0 -0
  33. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/dynamic_hdf5.py +0 -0
  34. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/file_arguments.py +0 -0
  35. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/h5_scan_map.py +0 -0
  36. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/redis_hdf5.py +0 -0
  37. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/static_hdf5.py +0 -0
  38. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/__init__.py +0 -0
  39. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/bliss.py +0 -0
  40. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/hdf5_retry.py +0 -0
  41. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/lima.py +0 -0
  42. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5api/utils/types.py +0 -0
  43. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/h5map/hdf5_items.py +0 -0
  44. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/lima/__init__.py +0 -0
  45. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/lima/client.py +0 -0
  46. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/py.typed +0 -0
  47. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/__init__.py +0 -0
  48. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/encoding/__init__.py +0 -0
  49. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/encoding/json.py +0 -0
  50. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/encoding/numeric.py +0 -0
  51. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/event_stream.py +0 -0
  52. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/exceptions.py +0 -0
  53. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/identities.py +0 -0
  54. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/redis_engine/scan.py +0 -0
  55. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/scan.py +0 -0
  56. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/schemas/__init__.py +0 -0
  57. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/schemas/scan_info.py +0 -0
  58. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/__init__.py +0 -0
  59. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/base.py +0 -0
  60. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/default.py +0 -0
  61. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/encoding/__init__.py +0 -0
  62. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/encoding/json.py +0 -0
  63. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/encoding/numeric.py +0 -0
  64. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/event_stream.py +0 -0
  65. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/hdf5_fallback/__init__.py +0 -0
  66. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/hdf5_fallback/stream.py +0 -0
  67. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/lima/__init__.py +0 -0
  68. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/lima/stream.py +0 -0
  69. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/scan_sequence/__init__.py +0 -0
  70. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/scan_sequence/stream.py +0 -0
  71. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/streams/sink.py +0 -0
  72. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/__init__.py +0 -0
  73. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/__init__.py +0 -0
  74. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/test_config.py +0 -0
  75. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/beacon/test_data.py +0 -0
  76. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/__init__.py +0 -0
  77. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/scanner.py +0 -0
  78. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_h5api_lima_utils.py +0 -0
  79. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_hdf5_handlers.py +0 -0
  80. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5api/test_static_files.py +0 -0
  81. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5map/__init__.py +0 -0
  82. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/h5map/test_hdf5_items.py +0 -0
  83. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/lima/__init__.py +0 -0
  84. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/__init__.py +0 -0
  85. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/conftest.py +0 -0
  86. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_cursor.py +0 -0
  87. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_scan.py +0 -0
  88. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/test_search.py +0 -0
  89. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/redis_engine/utils.py +0 -0
  90. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/encoding/__init__.py +0 -0
  91. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/encoding/test_json.py +0 -0
  92. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/encoding/test_numeric.py +0 -0
  93. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/__init__.py +0 -0
  94. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/test_file_backed_stream.py +0 -0
  95. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/plugins/test_plugin_base.py +0 -0
  96. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata/tests/streams/test_event_stream.py +0 -0
  97. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/SOURCES.txt +0 -0
  98. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/dependency_links.txt +0 -0
  99. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/entry_points.txt +0 -0
  100. {blissdata-2.3.0 → blissdata-2.4.0}/src/blissdata.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: blissdata
3
- Version: 2.3.0
3
+ Version: 2.4.0
4
4
  Summary: Bliss data streaming client
5
5
  Maintainer: BCU (ESRF)
6
6
  License-Expression: LGPL-3.0-or-later
@@ -19,9 +19,9 @@ Requires-Dist: h5py>=2.9
19
19
  Requires-Dist: msgpack
20
20
  Requires-Dist: numpy>=1.20
21
21
  Requires-Dist: numpydantic<1.6.10
22
- Requires-Dist: pydantic>=2.3.0
22
+ Requires-Dist: pydantic>=2.9
23
23
  Requires-Dist: redis>=4.6.0
24
- Requires-Dist: redis-om<1.0,>=0.3.1
24
+ Requires-Dist: redis-om~=1.0
25
25
  Requires-Dist: silx>=1.1.0
26
26
  Provides-Extra: tango
27
27
  Requires-Dist: pytango>=9.3.1; extra == "tango"
@@ -36,7 +36,7 @@ Requires-Dist: pytest-timeout; extra == "test"
36
36
  Requires-Dist: packaging; extra == "test"
37
37
  Provides-Extra: dev
38
38
  Requires-Dist: blissdata[test]; extra == "dev"
39
- Requires-Dist: black==22.3; extra == "dev"
39
+ Requires-Dist: black; extra == "dev"
40
40
  Requires-Dist: flake8>=4; extra == "dev"
41
41
  Requires-Dist: flake8-pep585; extra == "dev"
42
42
  Requires-Dist: mkdocs; extra == "dev"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "blissdata"
3
- version = "2.3.0"
3
+ version = "2.4.0"
4
4
  description = "Bliss data streaming client"
5
5
  readme = "README.md"
6
6
  license = "LGPL-3.0-or-later"
@@ -19,9 +19,9 @@ dependencies = [
19
19
  "msgpack",
20
20
  "numpy>=1.20",
21
21
  "numpydantic<1.6.10",
22
- "pydantic>=2.3.0", # (not a direct dep) redis-om 0.3.2 doesn't support lower versions
22
+ "pydantic>=2.9",
23
23
  "redis>=4.6.0",
24
- "redis-om>=0.3.1,<1.0",
24
+ "redis-om~=1.0",
25
25
  "silx>=1.1.0",
26
26
  ]
27
27
 
@@ -43,7 +43,7 @@ test = [
43
43
  ]
44
44
  dev = [
45
45
  "blissdata[test]",
46
- "black==22.3",
46
+ "black",
47
47
  "flake8>=4",
48
48
  "flake8-pep585",
49
49
  "mkdocs",
@@ -251,7 +251,7 @@ class DynamicHDF5Handler:
251
251
  try:
252
252
  return item.attrs[key] # try one more time
253
253
  except KeyError as e:
254
- if not _key_does_not_exist(e):
254
+ if not _attribute_does_not_exist(e):
255
255
  raise
256
256
  # attribute will never be created
257
257
  raise RetryTimeoutError(str(e)) from e
@@ -366,3 +366,14 @@ def _key_does_not_exist(exc: KeyError) -> bool:
366
366
 
367
367
  """
368
368
  return "doesn't exist" in str(exc)
369
+
370
+
371
+ def _attribute_does_not_exist(exc: KeyError) -> bool:
372
+ """
373
+ A real KeyError (meaning that the attribute does not exist) gives the following error message:
374
+
375
+ .. code
376
+
377
+ KeyError: "Unable to synchronously open attribute (can't locate attribute: 'myattr')"
378
+ """
379
+ return "can't locate attribute" in str(exc)
@@ -4,6 +4,7 @@ from .data_items import ( # noqa: F401
4
4
  InlineRaw,
5
5
  Inline,
6
6
  InlineBase64,
7
+ Compound,
7
8
  Stream,
8
9
  )
9
10
  from .hdf5_items import ( # noqa: F401
@@ -6,6 +6,7 @@ from pydantic import (
6
6
  Base64Bytes,
7
7
  Field,
8
8
  field_validator,
9
+ model_validator,
9
10
  model_serializer,
10
11
  Discriminator,
11
12
  Tag,
@@ -64,7 +65,13 @@ class InlineRaw(RootModel):
64
65
  def decode(self):
65
66
  value = self.model_dump()
66
67
  if value.dtype.kind in "UT":
67
- return np.vectorize(lambda x: x.encode(), otypes="O")(value)
68
+ # Make strings similar to those read back from h5py:
69
+ # - Convert to "object" dtype
70
+ # - Tell h5py that "object" is variable length string using
71
+ # metadata, so writing doesn't fail even in a compound dataset.
72
+ return np.vectorize(
73
+ lambda x: x.encode(), otypes=[np.dtype("O", metadata={"vlen": str})]
74
+ )(value)
68
75
  else:
69
76
  return value
70
77
 
@@ -155,14 +162,6 @@ class InlineBase64(BaseModel):
155
162
  return value
156
163
 
157
164
 
158
- class Stream(BaseModel):
159
- type: Literal["stream"] = "stream"
160
- stream: str
161
-
162
- def decode(self):
163
- raise TypeError("Stream model is just a reference, can't use decode()")
164
-
165
-
166
165
  def get_discriminator_value(v: Any) -> str:
167
166
  if isinstance(v, dict):
168
167
  return v.get("type")
@@ -171,12 +170,78 @@ def get_discriminator_value(v: Any) -> str:
171
170
  return "raw"
172
171
 
173
172
 
173
+ class Compound(BaseModel):
174
+ """Composite model to combine multiple DataItems as a structured array.
175
+ Unlike numpy structured arrays where data is a list of heterogeneous tuples,
176
+ here all elements of one dtype are stored together. This way existing
177
+ DataItem models can be reused as columns.
178
+
179
+ Validation ensures:
180
+ - Column names uniqueness
181
+ - Shape consistency between columns
182
+ """
183
+
184
+ type: Literal["compound"] = "compound"
185
+ columns: list[
186
+ tuple[
187
+ str,
188
+ Annotated[
189
+ (
190
+ Annotated[Inline, Tag("inline")]
191
+ | Annotated[InlineBase64, Tag("inline_b64")]
192
+ | Annotated[InlineRaw, Tag("raw")]
193
+ | Annotated["Compound", Tag("compound")]
194
+ # Note: Stream can't be part of a compound object
195
+ ),
196
+ Discriminator(get_discriminator_value),
197
+ ],
198
+ ]
199
+ ] = Field(min_length=1)
200
+ # Reusing DataItemType simplifies this model as it does not redefine
201
+ # handling of dtype, shape or base64 version of the values.
202
+ # Therefore the representation is the transposed of numpy one, which can be
203
+ # harder to read.
204
+
205
+ def decode(self):
206
+ decoded_columns = [(name, value.decode()) for name, value in self.columns]
207
+ rec = np.rec.fromarrays(
208
+ [value for _, value in decoded_columns],
209
+ names=[name for name, _ in decoded_columns],
210
+ )
211
+ return rec.view(np.ndarray)
212
+
213
+ @model_validator(mode="after")
214
+ @classmethod
215
+ def verify_names_and_shapes(cls, value):
216
+ """Ensure uniqueness of names and shape equality between columns"""
217
+ if len(value.columns) != len({name for name, _ in value.columns}):
218
+ raise ValueError("found columns with same name")
219
+
220
+ shape_iterator = iter((col.decode().shape for _, col in value.columns))
221
+ try:
222
+ first_shape = next(shape_iterator)
223
+ except StopIteration:
224
+ return value
225
+ if not all(first_shape == shape for shape in shape_iterator):
226
+ raise ValueError("columns have different shapes")
227
+ return value
228
+
229
+
230
+ class Stream(BaseModel):
231
+ type: Literal["stream"] = "stream"
232
+ stream: str
233
+
234
+ def decode(self):
235
+ raise TypeError("Stream model is just a reference, can't use decode()")
236
+
237
+
174
238
  DataItemType = Annotated[
175
239
  (
176
240
  Annotated[Inline, Tag("inline")]
177
241
  | Annotated[InlineBase64, Tag("inline_b64")]
178
242
  | Annotated[Stream, Tag("stream")]
179
243
  | Annotated[InlineRaw, Tag("raw")]
244
+ | Annotated[Compound, Tag("compound")]
180
245
  ),
181
246
  Discriminator(get_discriminator_value),
182
247
  ]
@@ -5,8 +5,8 @@
5
5
  # Copyright (c) Beamline Control Unit, ESRF
6
6
  # Distributed under the GNU LGPLv3. See LICENSE for more info.
7
7
 
8
- """Utility functions to read Lima images from file or device server.
9
- """
8
+ """Utility functions to read Lima images from file or device server."""
9
+
10
10
  import struct
11
11
  import numpy
12
12
  import typing
@@ -16,7 +16,6 @@ from dataclasses import dataclass
16
16
  from blissdata import DataStore, Scan, ScanState
17
17
  from blissdata.exceptions import NoScanAvailable, ScanNotFoundError, ScanLoadError
18
18
 
19
-
20
19
  memory_tracking_stream = "_MEMORY_TRACKING_"
21
20
  SEAL_ID = ("-".join([str(2**64 - 1)] * 2)).encode()
22
21
 
@@ -35,6 +34,7 @@ class NoFreeableMemoryError(Exception):
35
34
  class TrackedScan:
36
35
  scan: Scan
37
36
  last_activity: int
37
+ last_streams_ids: dict[str, int]
38
38
 
39
39
  def __eq__(self, other):
40
40
  return self.scan.key == other.scan.key
@@ -93,7 +93,7 @@ class MemoryTracker:
93
93
  scan._model.state_stream.key(), count=1
94
94
  )
95
95
  last_activity = int(raw[0][0].split(b"-")[0]) / 1000
96
- self._tracked_scans.add(TrackedScan(scan, last_activity))
96
+ self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
97
97
  except ScanNotFoundError:
98
98
  # scan already deleted from Redis by user, skip it
99
99
  pass
@@ -113,7 +113,7 @@ class MemoryTracker:
113
113
  try:
114
114
  scan = self._data_store.load_scan(scan_key)
115
115
  last_activity = int(since.split("-")[0]) / 1000
116
- self._tracked_scans.add(TrackedScan(scan, last_activity))
116
+ self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
117
117
  self._since = since
118
118
  logging.info(f"Start tracking {scan.key}")
119
119
  except ScanNotFoundError:
@@ -135,7 +135,14 @@ class MemoryTracker:
135
135
 
136
136
  if track.scan.state >= ScanState.PREPARED:
137
137
  streams_ids = self._data_streams_ids(track.scan)
138
- if streams_ids:
138
+ stream_activity = {
139
+ key: idx
140
+ for key, idx in streams_ids.items()
141
+ if key not in track.last_streams_ids
142
+ or track.last_streams_ids[key] != idx
143
+ }
144
+ if stream_activity:
145
+ track.last_streams_ids = streams_ids
139
146
  track.last_activity, _ = self._data_store._redis.time()
140
147
  all_data_streams_ids.update(streams_ids)
141
148
 
@@ -43,7 +43,7 @@ class DataStore:
43
43
  close_func = type(redis_pool).disconnect
44
44
  weakref.finalize(self, close_func, redis_pool)
45
45
 
46
- class StreamModel(HashModel):
46
+ class StreamModel(HashModel, index=True):
47
47
  """Actual stream model to be embedded into ScanModel, but not exposed to user directly.
48
48
  They are wrapped into Stream objects instead."""
49
49
 
@@ -60,7 +60,7 @@ class DataStore:
60
60
  encoding: dict | str | None = None
61
61
  info: dict | str | None = None
62
62
 
63
- class ScanModel(JsonModel):
63
+ class ScanModel(JsonModel, index=True):
64
64
  class Meta:
65
65
  global_key_prefix = identity_model_cls._meta.global_key_prefix
66
66
  model_key_prefix = "scan"
@@ -7,7 +7,6 @@
7
7
  import typing
8
8
  import typing_extensions
9
9
 
10
-
11
10
  ChannelRef = str
12
11
  """Identify a channel from the `scan_info["channels"]` field"""
13
12
 
@@ -23,7 +23,7 @@ def test_param_constructor(mocker):
23
23
 
24
24
 
25
25
  def test_get_file(mocker):
26
- RESPONSE1 = b"\x34\x00\x00\x00\x0E\x00\x00\x001|MY FILE DATA"
26
+ RESPONSE1 = b"\x34\x00\x00\x00\x0e\x00\x00\x001|MY FILE DATA"
27
27
  mock_socket = mocker.patch("socket.socket")
28
28
  mock_socket.return_value.recv.side_effect = [RESPONSE1]
29
29
  service = BeaconFiles("foo", 200)
@@ -35,7 +35,7 @@ def test_get_file(mocker):
35
35
 
36
36
 
37
37
  def test_get_tree(mocker):
38
- RESPONSE1 = b"""\x58\x00\x00\x00\x2C\x00\x00\x001|{"afile":null,"adir":{"anotherfile":null}}"""
38
+ RESPONSE1 = b"""\x58\x00\x00\x00\x2c\x00\x00\x001|{"afile":null,"adir":{"anotherfile":null}}"""
39
39
  mock_socket = mocker.patch("socket.socket")
40
40
  mock_socket.return_value.recv.side_effect = [RESPONSE1]
41
41
  service = BeaconFiles("foo", 200)
@@ -5,7 +5,6 @@ from pytest_redis.factories.proc import redis_proc
5
5
 
6
6
  from blissdata import DataStore
7
7
 
8
-
9
8
  redis_db = redis_proc(
10
9
  modules=[
11
10
  os.path.join(os.getenv("CONDA_PREFIX", "/usr"), "lib", "librejson.so"),
@@ -1,6 +1,10 @@
1
1
  import os
2
+ import time
3
+
4
+ import h5py
2
5
  import numpy
3
6
  import pytest
7
+
4
8
  from blissdata.h5api import dynamic_hdf5
5
9
  from . import scanner
6
10
 
@@ -325,3 +329,88 @@ def read_scan_data(nxroot, scan_number, counters, queue):
325
329
  numpy.testing.assert_array_equal(data, expected)
326
330
  scanner.log_point(scan_number, scan_index, queue, write=False)
327
331
  print(f"Finished reading scan {scan_number}")
332
+
333
+
334
+ @pytest.fixture
335
+ def h5file_finished_scan(tmp_path):
336
+ path = tmp_path / "test.h5"
337
+ with h5py.File(path, "w") as fh:
338
+ fh["/1.1/start_time"] = "2026-01-09T14:49:39.364371+01:00"
339
+ fh["/1.1/end_time"] = "2026-01-09T14:49:39.364371+01:00"
340
+ return str(path)
341
+
342
+
343
+ @pytest.fixture
344
+ def h5file_running_scan(tmp_path):
345
+ path = tmp_path / "test.h5"
346
+ with h5py.File(path, "w") as fh:
347
+ fh["/1.1/start_time"] = "2026-01-09T14:49:39.364371+01:00"
348
+ return str(path)
349
+
350
+
351
+ def test_non_existing_h5item_in_finished_scan(h5file_finished_scan):
352
+ _assert_non_existing_h5item(h5file_finished_scan, finished=True)
353
+
354
+
355
+ def test_non_existing_h5item_in_running_scan(h5file_running_scan):
356
+ _assert_non_existing_h5item(h5file_running_scan, finished=False)
357
+
358
+
359
+ def test_non_existing_h5attr_in_finished_scan(h5file_finished_scan):
360
+ _assert_non_existing_h5attr(h5file_finished_scan, finished=True)
361
+
362
+
363
+ def test_non_existing_h5attr_in_running_scan(h5file_running_scan):
364
+ _assert_non_existing_h5attr(h5file_running_scan, finished=False)
365
+
366
+
367
+ def _assert_non_existing_h5item(h5file, finished):
368
+ retry_timeout = 2
369
+ max_immediate_time = 0.5
370
+
371
+ with dynamic_hdf5.File(h5file, retry_timeout=retry_timeout) as fh:
372
+ scan = fh["/1.1"]
373
+
374
+ t0 = time.time()
375
+ with pytest.raises(KeyError):
376
+ scan["instrument"]
377
+ elapsed = time.time() - t0
378
+ if finished:
379
+ assert elapsed < max_immediate_time
380
+ else:
381
+ assert elapsed >= retry_timeout
382
+
383
+ t0 = time.time()
384
+ value = scan.get("instrument")
385
+ elapsed = time.time() - t0
386
+ assert value is None
387
+ if finished:
388
+ assert elapsed < max_immediate_time
389
+ else:
390
+ assert elapsed >= retry_timeout
391
+
392
+
393
+ def _assert_non_existing_h5attr(h5file, finished):
394
+ retry_timeout = 2
395
+ max_immediate_time = 0.5
396
+
397
+ with dynamic_hdf5.File(h5file, retry_timeout=retry_timeout) as fh:
398
+ scan = fh["/1.1"]
399
+
400
+ t0 = time.time()
401
+ with pytest.raises(KeyError):
402
+ scan.attrs["myattr"]
403
+ elapsed = time.time() - t0
404
+ if finished:
405
+ assert elapsed < max_immediate_time
406
+ else:
407
+ assert elapsed >= retry_timeout
408
+
409
+ t0 = time.time()
410
+ value = scan.attrs.get("myattr")
411
+ elapsed = time.time() - t0
412
+ assert value is None
413
+ if finished:
414
+ assert elapsed < max_immediate_time
415
+ else:
416
+ assert elapsed >= retry_timeout
@@ -307,6 +307,12 @@ def test_external_link(data_store, tmpdir):
307
307
  h5m.InlineRaw(["a", "b", "c"]),
308
308
  h5m.Inline(value=1.23, dtype="float16"),
309
309
  h5m.Inline(value=[1.23, "4.56"], dtype="float32"),
310
+ h5m.Compound(
311
+ columns=[
312
+ ("name", ["chan1", "chan2"]),
313
+ ("current", h5m.Inline(value=[1.23, "4.56"], dtype="float32")),
314
+ ]
315
+ ),
310
316
  ],
311
317
  )
312
318
  def test_static_dataset(data_store, tmpdir, value):
@@ -2,7 +2,7 @@ import pytest
2
2
  import pydantic
3
3
  import base64
4
4
  import numpy as np
5
- from blissdata.h5map import InlineRaw, Inline, InlineBase64
5
+ from blissdata.h5map import InlineRaw, Inline, InlineBase64, Compound, Stream
6
6
 
7
7
 
8
8
  @pytest.mark.parametrize(
@@ -211,6 +211,96 @@ def test_inline_load(json_input, expected_value):
211
211
  assert output.dtype == expected_value.dtype
212
212
 
213
213
 
214
+ @pytest.mark.parametrize(
215
+ "input_value, expected_json",
216
+ [
217
+ pytest.param(
218
+ [("a", 123)],
219
+ '{"type":"compound","columns":[["a",123]]}',
220
+ id="single column",
221
+ ),
222
+ pytest.param(
223
+ [("a", 1), ("b", 2)],
224
+ '{"type":"compound","columns":[["a",1],["b",2]]}',
225
+ id="scalars",
226
+ ),
227
+ pytest.param(
228
+ [("a", [1, 2]), ("b", [3.0, 4.0])],
229
+ '{"type":"compound","columns":[["a",[1,2]],["b",[3.0,4.0]]]}',
230
+ id="arrays",
231
+ ),
232
+ pytest.param(
233
+ [("a", [[1, 2], [3, 4]]), ("b", [[5.0, 6.0], [7.0, 8.0]])],
234
+ '{"type":"compound","columns":[["a",[[1,2],[3,4]]],["b",[[5.0,6.0],[7.0,8.0]]]]}',
235
+ id="ndarrays",
236
+ ),
237
+ pytest.param(
238
+ [("name", ["alice", "bob"]), ("id", [123, 456])],
239
+ '{"type":"compound","columns":[["name",["alice","bob"]],["id",[123,456]]]}',
240
+ id="strings",
241
+ ),
242
+ pytest.param(
243
+ [
244
+ ("a", Inline(value=[1.2, 2.3], dtype="float16")),
245
+ ("b", Inline(value=[12, 23], dtype="uint32")),
246
+ ],
247
+ '{"type":"compound","columns":[["a",{"type":"inline","dtype":"float16","value":[1.2,2.3]}],["b",{"type":"inline","dtype":"uint32","value":[12,23]}]]}',
248
+ id="explicit types",
249
+ ),
250
+ pytest.param(
251
+ [
252
+ ["id", [1, 2, 3]],
253
+ [
254
+ "person",
255
+ Compound(
256
+ columns=[
257
+ ["name", ["Alice", "Bob", "Charlie"]],
258
+ [
259
+ "email",
260
+ [
261
+ "alice@example.com",
262
+ "bob@example.com",
263
+ "charlie@example.com",
264
+ ],
265
+ ],
266
+ ]
267
+ ),
268
+ ],
269
+ ],
270
+ '{"type":"compound","columns":[["id",[1,2,3]],["person",{"type":"compound","columns":[["name",["Alice","Bob","Charlie"]],["email",["alice@example.com","bob@example.com","charlie@example.com"]]]}]]}',
271
+ id="nested",
272
+ ),
273
+ ],
274
+ )
275
+ def test_compound(input_value, expected_json):
276
+ model = Compound(columns=input_value)
277
+ assert model.model_dump_json() == expected_json
278
+ _ = model.decode()
279
+
280
+
281
+ @pytest.mark.parametrize(
282
+ "input_value",
283
+ [
284
+ pytest.param([], id="empty"),
285
+ pytest.param(
286
+ [("aaa", [1, 2]), ("bbb", ["hello", "bye"]), ("ccc", [1.23])],
287
+ id="misaligned",
288
+ ),
289
+ pytest.param(
290
+ [("stream", Stream(stream="stream_name")), ("integer", 42)],
291
+ id="stream",
292
+ ),
293
+ pytest.param(
294
+ [("twin", 1), ("twin", 2)],
295
+ id="duplicate name",
296
+ ),
297
+ ],
298
+ )
299
+ def test_compound_forbidden_value(input_value):
300
+ with pytest.raises(pydantic.ValidationError):
301
+ _ = Compound(columns=input_value)
302
+
303
+
214
304
  @pytest.mark.parametrize(
215
305
  "json_input, expected_value",
216
306
  [
@@ -63,13 +63,13 @@ RAW_YUV422PACKED_VIDEO = (
63
63
  RAW_RGB24_VIDEO = (
64
64
  b"VDEO\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
65
65
  \x00\x02\x00\x00\x00\x02\x00\x00\x00 \x00\x00\x00\x00"
66
- + b"\xFF\x00\x00\x00\xFF\x00\x00\x00\xFF\x00\x00\x00"
66
+ + b"\xff\x00\x00\x00\xff\x00\x00\x00\xff\x00\x00\x00"
67
67
  )
68
68
 
69
69
  RAW_RGB32_VIDEO = (
70
70
  b"VDEO\x00\x01\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\
71
71
  \x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00 \x00\x00\x00\x00"
72
- + b"\xFF\x00\x00\x00\x00\xFF\x00\x00\x00\x00\xFF\x00\x00\x00\x00\xFF"
72
+ + b"\xff\x00\x00\x00\x00\xff\x00\x00\x00\x00\xff\x00\x00\x00\x00\xff"
73
73
  )
74
74
 
75
75
  RAW_BAYER_RG16 = (
@@ -224,29 +224,35 @@ def test_delete_inactive_scan(redis_url, data_store, dummy_id):
224
224
  active_scan = data_store.create_scan(dummy_id)
225
225
 
226
226
  stream_definition = Stream.make_definition("mystream", "int")
227
- stream = active_scan.create_stream(stream_definition)
227
+ stream_ina = inactive_scan.create_stream(stream_definition)
228
228
 
229
+ stream_definition = Stream.make_definition("mystream", "int")
230
+ stream_act = active_scan.create_stream(stream_definition)
231
+
232
+ inactive_scan.prepare()
233
+ inactive_scan.start()
229
234
  active_scan.prepare()
230
235
  active_scan.start()
231
236
 
232
- # wait for tracker to find the scan
237
+ # wait for tracker to find the scans
233
238
  nb_scans_to_track = 2
234
239
  while nb_scans_to_track:
235
- stream.send(1)
240
+ stream_ina.send(1)
241
+ stream_act.send(1)
236
242
  line = proc.stdout.readline()
237
243
  print(line, end="")
238
244
  if "Start tracking" in line:
239
245
  nb_scans_to_track -= 1
240
246
 
241
247
  while True:
242
- stream.send(2)
248
+ stream_act.send(2)
243
249
  line = proc.stdout.readline()
244
250
  print(line, end="")
245
251
  if f"{inactive_scan.key} was inactive" in line:
246
252
  break
247
253
 
248
254
  for _ in range(4):
249
- stream.send(3)
255
+ stream_act.send(3)
250
256
  line = proc.stdout.readline()
251
257
  print(line, end="")
252
258
  assert f"{active_scan.key} was inactive" not in line
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: blissdata
3
- Version: 2.3.0
3
+ Version: 2.4.0
4
4
  Summary: Bliss data streaming client
5
5
  Maintainer: BCU (ESRF)
6
6
  License-Expression: LGPL-3.0-or-later
@@ -19,9 +19,9 @@ Requires-Dist: h5py>=2.9
19
19
  Requires-Dist: msgpack
20
20
  Requires-Dist: numpy>=1.20
21
21
  Requires-Dist: numpydantic<1.6.10
22
- Requires-Dist: pydantic>=2.3.0
22
+ Requires-Dist: pydantic>=2.9
23
23
  Requires-Dist: redis>=4.6.0
24
- Requires-Dist: redis-om<1.0,>=0.3.1
24
+ Requires-Dist: redis-om~=1.0
25
25
  Requires-Dist: silx>=1.1.0
26
26
  Provides-Extra: tango
27
27
  Requires-Dist: pytango>=9.3.1; extra == "tango"
@@ -36,7 +36,7 @@ Requires-Dist: pytest-timeout; extra == "test"
36
36
  Requires-Dist: packaging; extra == "test"
37
37
  Provides-Extra: dev
38
38
  Requires-Dist: blissdata[test]; extra == "dev"
39
- Requires-Dist: black==22.3; extra == "dev"
39
+ Requires-Dist: black; extra == "dev"
40
40
  Requires-Dist: flake8>=4; extra == "dev"
41
41
  Requires-Dist: flake8-pep585; extra == "dev"
42
42
  Requires-Dist: mkdocs; extra == "dev"
@@ -4,9 +4,9 @@ h5py>=2.9
4
4
  msgpack
5
5
  numpy>=1.20
6
6
  numpydantic<1.6.10
7
- pydantic>=2.3.0
7
+ pydantic>=2.9
8
8
  redis>=4.6.0
9
- redis-om<1.0,>=0.3.1
9
+ redis-om~=1.0
10
10
  silx>=1.1.0
11
11
 
12
12
  [beacon]
@@ -14,7 +14,7 @@ ruamel.yaml
14
14
 
15
15
  [dev]
16
16
  blissdata[test]
17
- black==22.3
17
+ black
18
18
  flake8>=4
19
19
  flake8-pep585
20
20
  mkdocs
File without changes
File without changes
File without changes