blissdata 2.3.0__tar.gz → 2.4.1__tar.gz

{blissdata-2.3.0/src/blissdata.egg-info → blissdata-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blissdata
-Version: 2.3.0
+Version: 2.4.1
 Summary: Bliss data streaming client
 Maintainer: BCU (ESRF)
 License-Expression: LGPL-3.0-or-later
@@ -19,9 +19,9 @@ Requires-Dist: h5py>=2.9
 Requires-Dist: msgpack
 Requires-Dist: numpy>=1.20
 Requires-Dist: numpydantic<1.6.10
-Requires-Dist: pydantic>=2.3.0
+Requires-Dist: pydantic>=2.9
 Requires-Dist: redis>=4.6.0
-Requires-Dist: redis-om<1.0,>=0.3.1
+Requires-Dist: redis-om~=1.0
 Requires-Dist: silx>=1.1.0
 Provides-Extra: tango
 Requires-Dist: pytango>=9.3.1; extra == "tango"
@@ -36,7 +36,7 @@ Requires-Dist: pytest-timeout; extra == "test"
 Requires-Dist: packaging; extra == "test"
 Provides-Extra: dev
 Requires-Dist: blissdata[test]; extra == "dev"
-Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: black; extra == "dev"
 Requires-Dist: flake8>=4; extra == "dev"
 Requires-Dist: flake8-pep585; extra == "dev"
 Requires-Dist: mkdocs; extra == "dev"

{blissdata-2.3.0 → blissdata-2.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blissdata"
-version = "2.3.0"
+version = "2.4.1"
 description = "Bliss data streaming client"
 readme = "README.md"
 license = "LGPL-3.0-or-later"
@@ -19,9 +19,9 @@ dependencies = [
     "msgpack",
     "numpy>=1.20",
     "numpydantic<1.6.10",
-    "pydantic>=2.3.0", # (not a direct dep) redis-om 0.3.2 doesn't support lower versions
+    "pydantic>=2.9",
     "redis>=4.6.0",
-    "redis-om>=0.3.1,<1.0",
+    "redis-om~=1.0",
     "silx>=1.1.0",
 ]
 
@@ -43,7 +43,7 @@ test = [
 ]
 dev = [
     "blissdata[test]",
-    "black==22.3",
+    "black",
     "flake8>=4",
     "flake8-pep585",
     "mkdocs",

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/h5api/utils/hdf5.py

@@ -251,7 +251,7 @@ class DynamicHDF5Handler:
             try:
                 return item.attrs[key]  # try one more time
             except KeyError as e:
-                if not _key_does_not_exist(e):
+                if not _attribute_does_not_exist(e):
                     raise
                 # attribute will never be created
                 raise RetryTimeoutError(str(e)) from e
@@ -366,3 +366,14 @@ def _key_does_not_exist(exc: KeyError) -> bool:
 
     """
     return "doesn't exist" in str(exc)
+
+
+def _attribute_does_not_exist(exc: KeyError) -> bool:
+    """
+    A real KeyError (meaning that the attribute does not exist) gives the following error message:
+
+    .. code
+
+        KeyError: "Unable to synchronously open attribute (can't locate attribute: 'myattr')"
+    """
+    return "can't locate attribute" in str(exc)

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/h5map/__init__.py

@@ -4,6 +4,7 @@ from .data_items import (  # noqa: F401
     InlineRaw,
     Inline,
     InlineBase64,
+    Compound,
     Stream,
 )
 from .hdf5_items import (  # noqa: F401

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/h5map/data_items.py

@@ -6,6 +6,7 @@ from pydantic import (
     Base64Bytes,
     Field,
     field_validator,
+    model_validator,
     model_serializer,
     Discriminator,
     Tag,
@@ -64,7 +65,13 @@ class InlineRaw(RootModel):
     def decode(self):
         value = self.model_dump()
         if value.dtype.kind in "UT":
-            return np.vectorize(lambda x: x.encode(), otypes="O")(value)
+            # Make strings similar to those read back from h5py:
+            #   - Convert to "object" dtype
+            #   - Tell h5py that "object" is variable length string using
+            #     metadata, so writing doesn't fail even in a compound dataset.
+            return np.vectorize(
+                lambda x: x.encode(), otypes=[np.dtype("O", metadata={"vlen": str})]
+            )(value)
         else:
             return value
 
@@ -155,14 +162,6 @@ class InlineBase64(BaseModel):
             return value
 
 
-class Stream(BaseModel):
-    type: Literal["stream"] = "stream"
-    stream: str
-
-    def decode(self):
-        raise TypeError("Stream model is just a reference, can't use decode()")
-
-
 def get_discriminator_value(v: Any) -> str:
     if isinstance(v, dict):
         return v.get("type")
@@ -171,12 +170,78 @@ def get_discriminator_value(v: Any) -> str:
     return "raw"
 
 
+class Compound(BaseModel):
+    """Composite model to combine multiple DataItems as a structured array.
+    Unlike numpy structured arrays where data is a list of heterogeneous tuples,
+    here all elements of one dtype are stored together. This way existing
+    DataItem models can be reused as columns.
+
+    Validation ensures:
+    - Column names uniqueness
+    - Shape consistency between columns
+    """
+
+    type: Literal["compound"] = "compound"
+    columns: list[
+        tuple[
+            str,
+            Annotated[
+                (
+                    Annotated[Inline, Tag("inline")]
+                    | Annotated[InlineBase64, Tag("inline_b64")]
+                    | Annotated[InlineRaw, Tag("raw")]
+                    | Annotated["Compound", Tag("compound")]
+                    # Note: Stream can't be part of a compound object
+                ),
+                Discriminator(get_discriminator_value),
+            ],
+        ]
+    ] = Field(min_length=1)
+    # Reusing DataItemType simplifies this model as it does not redefine
+    # handling of dtype, shape or base64 version of the values.
+    # Therefore the representation is the transposed of numpy one, which can be
+    # harder to read.
+
+    def decode(self):
+        decoded_columns = [(name, value.decode()) for name, value in self.columns]
+        rec = np.rec.fromarrays(
+            [value for _, value in decoded_columns],
+            names=[name for name, _ in decoded_columns],
+        )
+        return rec.view(np.ndarray)
+
+    @model_validator(mode="after")
+    @classmethod
+    def verify_names_and_shapes(cls, value):
+        """Ensure uniqueness of names and shape equality between columns"""
+        if len(value.columns) != len({name for name, _ in value.columns}):
+            raise ValueError("found columns with same name")
+
+        shape_iterator = iter((col.decode().shape for _, col in value.columns))
+        try:
+            first_shape = next(shape_iterator)
+        except StopIteration:
+            return value
+        if not all(first_shape == shape for shape in shape_iterator):
+            raise ValueError("columns have different shapes")
+        return value
+
+
+class Stream(BaseModel):
+    type: Literal["stream"] = "stream"
+    stream: str
+
+    def decode(self):
+        raise TypeError("Stream model is just a reference, can't use decode()")
+
+
 DataItemType = Annotated[
     (
         Annotated[Inline, Tag("inline")]
         | Annotated[InlineBase64, Tag("inline_b64")]
         | Annotated[Stream, Tag("stream")]
         | Annotated[InlineRaw, Tag("raw")]
+        | Annotated[Compound, Tag("compound")]
     ),
     Discriminator(get_discriminator_value),
 ]

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/lima/client.py

@@ -5,13 +5,13 @@
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 import math
+import time
 import logging
 import numpy as np
 from packaging.version import Version
 from functools import cached_property
 from dataclasses import dataclass
 
-from blissdata.exceptions import IndexNoMoreThereError, IndexNotYetThereError
 from . import image_utils
 
 try:
@@ -58,7 +58,7 @@ class ImageReference:
 class LimaClient:
     PROTOCOL_VERSION = 1
 
-    def __init__(self, data_store, **lima_info):
+    def __init__(self, data_store, request_update_cb, **lima_info):
         protocol = lima_info["protocol_version"]
         if self.PROTOCOL_VERSION != protocol:
             raise Exception(
@@ -66,6 +66,8 @@ class LimaClient:
             )
 
         self._data_store = data_store
+        self._request_update = request_update_cb
+        self._stream_sealed = False
         self._proxy = None
         self._server_url = lima_info["server_url"]
         self._buffer_max_number = lima_info["buffer_max_number"]
@@ -123,9 +125,10 @@ class LimaClient:
     def __len__(self):
         return self._last_index + 1
 
-    def update(self, last_index, last_index_saved):
+    def update(self, last_index=-1, last_index_saved=-1, sealed=False):
         self._last_index = max(self._last_index, last_index)
         self._last_index_saved = max(self._last_index_saved, last_index_saved)
+        self._stream_sealed = self._stream_sealed or sealed
         if self._saved:
             self._last_index_saved_and_closed = self._last_readable_index()
 
@@ -155,10 +158,14 @@ class LimaClient:
 
     def _last_readable_index(self):
         """Return the last frame index that can be read from a closed file, assuming a
-        file is closed when at least one frame is saved into the next one.
+        file is closed either when stream is sealed or when at least one frame is saved
+        into the next file.
         NOTE: This is trickier than a modulo as an acquisition may not be an exact
         multiple of the file length (see self._decompose_frame_id).
         """
+        if self._stream_sealed:
+            return self._last_index_saved
+
         acq_in_scan, file_in_acq, _ = self._decompose_frame_id(self._last_index_saved)
         return (
             acq_in_scan * self._frames_per_acquisition
@@ -215,7 +222,7 @@ class LimaClient:
         frame_id = normalize_frame_id(data.frame_id)
         return image_utils.ImageData(data.array, frame_id, None)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: int | slice):
         if isinstance(key, int):
             return self._get_index(key)
         elif isinstance(key, slice):
@@ -232,21 +239,28 @@ class LimaClient:
             raise NotImplementedError("Negative index not supported")
 
         if self._saved and frame_id <= self._last_index_saved_and_closed:
-            # immediately accessible from file
+            # 1. priority to file if available
             return self._get_from_file(frame_id)
         elif frame_id <= self._last_index:
-            # try to access on lima server
             try:
+                # 2. try from server then
                 return self._get_from_server(frame_id)
-            except IndexNoMoreThereError:
-                # wait for image to become available from file
-                # TODO wait_for_file(timeout)
+            except IndexError:
                 if self._saved:
+                    # 3. finally wait for file
+                    while (
+                        frame_id > self._last_index_saved_and_closed
+                        and not self._stream_sealed
+                    ):
+                        self._request_update()
+                        time.sleep(0.1)
+                    if frame_id > self._last_index_saved_and_closed:
+                        raise
                     return self._get_from_file(frame_id)
                 else:
                     raise
         else:
-            raise IndexNotYetThereError
+            raise IndexError
 
     def _get_slice(self, key):
         key_range = range(*key.indices(len(self)))
@@ -267,7 +281,10 @@ class LimaClient:
 
     def _get_from_server(self, frame_id):
         if self._buffer_max_number <= (self._last_index - frame_id):
-            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+            # if frame_id is in the current acquisition, then it's not in the
+            # buffer any longer. If another acquisition started, it's not here
+            # neither.
+            raise IndexError
 
         acq_in_scan, file_in_acq, frame_in_file = self._decompose_frame_id(frame_id)
         frame_in_acq = file_in_acq * self._frames_per_file + frame_in_file
@@ -278,15 +295,24 @@ class LimaClient:
         if not self._is_acquisition_still_on_server(
             acq_in_scan + self._acquisition_offset
         ):
-            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+            raise IndexError
 
-        frame = image_utils.image_from_server(self.proxy, frame_in_acq)
-        assert frame_in_acq == frame.frame_id
+        try:
+            frame = image_utils.image_from_server(self.proxy, frame_in_acq)
+        except (RuntimeError, image_utils.NoImageAvailable) as e:
+            raise IndexError from e
+
+        if frame_in_acq != frame.frame_id:
+            _logger.warning(
+                f"Requested frame {frame_in_acq} from lima device, "
+                f"got frame {frame.frame_id}. Discarding it."
+            )
+            raise IndexError
 
         if not self._is_acquisition_still_on_server(
             acq_in_scan + self._acquisition_offset
         ):
-            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+            raise IndexError
 
         return frame.array
 
@@ -296,19 +322,13 @@ class LimaClient:
             ref.file_path, ref.data_path, ref.index, ref.format
         )
 
-    def get_references(self, key) -> ImageReference | list[ImageReference]:
-        # NOTE: _last_index is used to define when references are available.
-        # It should normally be _last_index_saved instead, but then, a LimaView
-        # would not return the same amount of images and references.
-        saved_len = self._last_index + 1
+    def get_references(self, key: int | slice) -> ImageReference | list[ImageReference]:
+        """Create references no matter the images are available or not. It is
+        caller responsibility to query indices that make sense."""
         if isinstance(key, int):
-            if key < 0:
-                key += saved_len
-            elif key >= saved_len:
-                raise IndexNotYetThereError
             return self._file_ref(key)
         elif isinstance(key, slice):
-            key_range = range(*key.indices(saved_len))
+            key_range = range(*key.indices(len(self)))
             return [self._file_ref(i) for i in key_range]
         else:
             raise TypeError(

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/lima/image_utils.py

@@ -5,8 +5,8 @@
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 
-"""Utility functions to read Lima images from file or device server.
-"""
+"""Utility functions to read Lima images from file or device server."""
+
 import struct
 import numpy
 import typing

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/redis_engine/memory_tracker.py

@@ -16,7 +16,6 @@ from dataclasses import dataclass
 from blissdata import DataStore, Scan, ScanState
 from blissdata.exceptions import NoScanAvailable, ScanNotFoundError, ScanLoadError
 
-
 memory_tracking_stream = "_MEMORY_TRACKING_"
 SEAL_ID = ("-".join([str(2**64 - 1)] * 2)).encode()
 
@@ -35,6 +34,7 @@ class NoFreeableMemoryError(Exception):
 class TrackedScan:
     scan: Scan
     last_activity: int
+    last_streams_ids: dict[str, int]
 
     def __eq__(self, other):
         return self.scan.key == other.scan.key
@@ -93,7 +93,7 @@ class MemoryTracker:
                     scan._model.state_stream.key(), count=1
                 )
                 last_activity = int(raw[0][0].split(b"-")[0]) / 1000
-                self._tracked_scans.add(TrackedScan(scan, last_activity))
+                self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
             except ScanNotFoundError:
                 # scan already deleted from Redis by user, skip it
                 pass
@@ -113,7 +113,7 @@ class MemoryTracker:
                 try:
                     scan = self._data_store.load_scan(scan_key)
                     last_activity = int(since.split("-")[0]) / 1000
-                    self._tracked_scans.add(TrackedScan(scan, last_activity))
+                    self._tracked_scans.add(TrackedScan(scan, last_activity, {}))
                     self._since = since
                     logging.info(f"Start tracking {scan.key}")
                 except ScanNotFoundError:
@@ -135,7 +135,14 @@ class MemoryTracker:
 
             if track.scan.state >= ScanState.PREPARED:
                 streams_ids = self._data_streams_ids(track.scan)
-                if streams_ids:
+                stream_activity = {
+                    key: idx
+                    for key, idx in streams_ids.items()
+                    if key not in track.last_streams_ids
+                    or track.last_streams_ids[key] != idx
+                }
+                if stream_activity:
+                    track.last_streams_ids = streams_ids
                     track.last_activity, _ = self._data_store._redis.time()
                 all_data_streams_ids.update(streams_ids)
 

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/redis_engine/store.py

@@ -43,7 +43,7 @@ class DataStore:
         close_func = type(redis_pool).disconnect
         weakref.finalize(self, close_func, redis_pool)
 
-        class StreamModel(HashModel):
+        class StreamModel(HashModel, index=True):
             """Actual stream model to be embedded into ScanModel, but not exposed to user directly.
             They are wrapped into Stream objects instead."""
 
@@ -60,7 +60,7 @@ class DataStore:
             encoding: dict | str | None = None
             info: dict | str | None = None
 
-        class ScanModel(JsonModel):
+        class ScanModel(JsonModel, index=True):
             class Meta:
                 global_key_prefix = identity_model_cls._meta.global_key_prefix
                 model_key_prefix = "scan"

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/schemas/scan_info_display.py

@@ -7,7 +7,6 @@
 import typing
 import typing_extensions
 
-
 ChannelRef = str
 """Identify a channel from the `scan_info["channels"]` field"""
 

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/streams/lima/stream.py

@@ -28,8 +28,8 @@ from blissdata.lima.image_utils import ImageData
 
 
 class LimaView(BaseView):
-    def __init__(self, client, start, stop):
-        self._client = client
+    def __init__(self, stream: "LimaStream", start: int, stop: int) -> None:
+        self._stream = stream
         self._view_range = range(start, stop)
 
     @property
@@ -41,13 +41,13 @@ class LimaView(BaseView):
 
     def get_data(self, start=None, stop=None):
         trimmed_range = self._view_range[start:stop]
-        return self._client[trimmed_range.start : trimmed_range.stop]
+        return self._stream[trimmed_range.start : trimmed_range.stop]
 
     def get_references(
         self, start=None, stop=None
     ) -> ImageReference | list[ImageReference]:
         trimmed_range = self._view_range[start:stop]
-        return self._client.get_references(
+        return self._stream.get_references(
             slice(trimmed_range.start, trimmed_range.stop)
         )
 
@@ -67,10 +67,16 @@ class LimaStream(BaseStream, LimaDirectAccess):
     def __init__(self, event_stream):
         super().__init__(event_stream)
         self._client = LimaClient(
-            event_stream._data_store, **event_stream.info["lima_info"]
+            data_store=event_stream._data_store,
+            request_update_cb=self._force_update_client,
+            **event_stream.info["lima_info"],
         )
         self._cursor = Stream(event_stream).cursor()
 
+    def _force_update_client(self):
+        _, status_list, sealed = self._event_stream._read_last(0)
+        self._client.update(**status_list[0], sealed=sealed)
+
     @staticmethod
     def make_definition(
         name: str,
@@ -148,28 +154,41 @@ class LimaStream(BaseStream, LimaDirectAccess):
         self._update_client()
         return len(self._client)
 
-    def _update_client(self):
+    def _update_client_if_needed(self, key: int | slice):
+        if isinstance(key, slice):
+            need_update = key.stop is None or not (0 <= key.stop <= len(self._client))
+        else:
+            need_update = not (0 <= key < len(self._client))
+        if not need_update:
+            return
+
         try:
             view = self._cursor.read(block=False, last_only=True)
         except EndOfStream:
-            return
-        if view is not None:
+            view = None
+        # use ._seal instead of .is_sealed() to skip redis query and because
+        # we just updated it by reading cursor.
+        sealed = self._event_stream._seal is not None
+        if view is None:
+            self._client.update(sealed=sealed)
+        else:
             last_status = view.get_data(-1, None)[0]
-            self._client.update(**last_status)
+            self._client.update(**last_status, sealed=sealed)
 
     def __getitem__(self, key):
-        if isinstance(key, slice):
-            need_update = key.stop is None or not (0 <= key.stop < len(self._client))
-        else:
-            need_update = not (0 <= key < len(self._client))
+        return self._get_from_client(key, reference_mode=False)
 
-        if need_update:
-            self._update_client()
+    def get_references(self, key) -> ImageReference | list[ImageReference]:
+        return self._get_from_client(key, reference_mode=True)
 
+    def _get_from_client(self, key, reference_mode):
+        self._update_client_if_needed(key)
         try:
-            return self._client[key]
+            if reference_mode:
+                return self._client.get_references(key)
+            else:
+                return self._client[key]
         except IndexError:
-            # TODO could be verified before asking the client
             index = key.start if isinstance(key, slice) else key
             if index is None:
                 index = 0
@@ -177,7 +196,7 @@ class LimaStream(BaseStream, LimaDirectAccess):
                 index += len(self._client)
 
             if index >= len(self._client):
-                if self._event_stream.is_sealed():
+                if self._client._stream_sealed:
                     raise IndexWontBeThereError
                 else:
                     raise IndexNotYetThereError
@@ -189,7 +208,7 @@ class LimaStream(BaseStream, LimaDirectAccess):
         return True
 
     def _build_view_from_events(self, index: int, events: EventRange, last_only: bool):
-        self._client.update(**events.data[-1])
+        self._client.update(**events.data[-1], sealed=events.end_of_stream)
 
         if len(self._client) <= index:
             # no new images despite client update
@@ -199,34 +218,7 @@ class LimaStream(BaseStream, LimaDirectAccess):
             start = len(self._client) - 1
         else:
             start = index
-        return LimaView(self._client, start, len(self._client))
-
-    def get_references(self, key) -> ImageReference | list[ImageReference]:
-        if isinstance(key, slice):
-            need_update = key.stop is None or not (0 <= key.stop < len(self._client))
-        else:
-            need_update = not (0 <= key < len(self._client))
-
-        if need_update:
-            self._update_client()
-
-        try:
-            return self._client.get_references(key)
-        except IndexError:
-            # TODO could be verified before asking the client
-            index = key.start if isinstance(key, slice) else key
-            if index is None:
-                index = 0
-            elif index < 0:
-                index += len(self._client)
-
-            if index >= len(self._client):
-                if self._event_stream.is_sealed():
-                    raise IndexWontBeThereError
-                else:
-                    raise IndexNotYetThereError
-            else:
-                raise IndexNoMoreThereError
+        return LimaView(self, start, len(self._client))
 
     def get_last_live_image(self) -> ImageData:
         return self._client.get_last_live_image()

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/tests/beacon/test_files.py

@@ -23,7 +23,7 @@ def test_param_constructor(mocker):
 
 
 def test_get_file(mocker):
-    RESPONSE1 = b"\x34\x00\x00\x00\x0E\x00\x00\x001|MY FILE DATA"
+    RESPONSE1 = b"\x34\x00\x00\x00\x0e\x00\x00\x001|MY FILE DATA"
     mock_socket = mocker.patch("socket.socket")
     mock_socket.return_value.recv.side_effect = [RESPONSE1]
     service = BeaconFiles("foo", 200)
@@ -35,7 +35,7 @@ def test_get_file(mocker):
 
 
 def test_get_tree(mocker):
-    RESPONSE1 = b"""\x58\x00\x00\x00\x2C\x00\x00\x001|{"afile":null,"adir":{"anotherfile":null}}"""
+    RESPONSE1 = b"""\x58\x00\x00\x00\x2c\x00\x00\x001|{"afile":null,"adir":{"anotherfile":null}}"""
     mock_socket = mocker.patch("socket.socket")
     mock_socket.return_value.recv.side_effect = [RESPONSE1]
     service = BeaconFiles("foo", 200)

{blissdata-2.3.0 → blissdata-2.4.1}/src/blissdata/tests/conftest.py

@@ -5,7 +5,6 @@ from pytest_redis.factories.proc import redis_proc
 
 from blissdata import DataStore
 
-
 redis_db = redis_proc(
     modules=[
         os.path.join(os.getenv("CONDA_PREFIX", "/usr"), "lib", "librejson.so"),