blissdata 2.4.0__tar.gz → 2.4.2__tar.gz

{blissdata-2.4.0/src/blissdata.egg-info → blissdata-2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blissdata
-Version: 2.4.0
+Version: 2.4.2
 Summary: Bliss data streaming client
 Maintainer: BCU (ESRF)
 License-Expression: LGPL-3.0-or-later

{blissdata-2.4.0 → blissdata-2.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blissdata"
-version = "2.4.0"
+version = "2.4.2"
 description = "Bliss data streaming client"
 readme = "README.md"
 license = "LGPL-3.0-or-later"

{blissdata-2.4.0 → blissdata-2.4.2}/src/blissdata/lima/client.py

@@ -5,13 +5,13 @@
 # Copyright (c) Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 import math
+import time
 import logging
 import numpy as np
 from packaging.version import Version
 from functools import cached_property
 from dataclasses import dataclass
 
-from blissdata.exceptions import IndexNoMoreThereError, IndexNotYetThereError
 from . import image_utils
 
 try:
@@ -58,7 +58,7 @@ class ImageReference:
 class LimaClient:
     PROTOCOL_VERSION = 1
 
-    def __init__(self, data_store, **lima_info):
+    def __init__(self, data_store, request_update_cb, **lima_info):
         protocol = lima_info["protocol_version"]
         if self.PROTOCOL_VERSION != protocol:
             raise Exception(
@@ -66,6 +66,8 @@ class LimaClient:
             )
 
         self._data_store = data_store
+        self._request_update = request_update_cb
+        self._stream_sealed = False
         self._proxy = None
         self._server_url = lima_info["server_url"]
         self._buffer_max_number = lima_info["buffer_max_number"]
@@ -123,9 +125,10 @@ class LimaClient:
     def __len__(self):
         return self._last_index + 1
 
-    def update(self, last_index, last_index_saved):
+    def update(self, last_index=-1, last_index_saved=-1, sealed=False):
         self._last_index = max(self._last_index, last_index)
         self._last_index_saved = max(self._last_index_saved, last_index_saved)
+        self._stream_sealed = self._stream_sealed or sealed
         if self._saved:
             self._last_index_saved_and_closed = self._last_readable_index()
 
@@ -155,10 +158,14 @@ class LimaClient:
 
     def _last_readable_index(self):
         """Return the last frame index that can be read from a closed file, assuming a
-        file is closed when at least one frame is saved into the next one.
+        file is closed either when stream is sealed or when at least one frame is saved
+        into the next file.
         NOTE: This is trickier than a modulo as an acquisition may not be an exact
         multiple of the file length (see self._decompose_frame_id).
         """
+        if self._stream_sealed:
+            return self._last_index_saved
+
         acq_in_scan, file_in_acq, _ = self._decompose_frame_id(self._last_index_saved)
         return (
             acq_in_scan * self._frames_per_acquisition
@@ -215,7 +222,7 @@ class LimaClient:
         frame_id = normalize_frame_id(data.frame_id)
         return image_utils.ImageData(data.array, frame_id, None)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: int | slice):
         if isinstance(key, int):
             return self._get_index(key)
         elif isinstance(key, slice):
@@ -232,21 +239,28 @@ class LimaClient:
             raise NotImplementedError("Negative index not supported")
 
         if self._saved and frame_id <= self._last_index_saved_and_closed:
-            # immediately accessible from file
+            # 1. priority to file if available
             return self._get_from_file(frame_id)
         elif frame_id <= self._last_index:
-            # try to access on lima server
             try:
+                # 2. try from server then
                 return self._get_from_server(frame_id)
-            except IndexNoMoreThereError:
-                # wait for image to become available from file
-                # TODO wait_for_file(timeout)
+            except IndexError:
                 if self._saved:
+                    # 3. finally wait for file
+                    while (
+                        frame_id > self._last_index_saved_and_closed
+                        and not self._stream_sealed
+                    ):
+                        self._request_update()
+                        time.sleep(0.1)
+                    if frame_id > self._last_index_saved_and_closed:
+                        raise
                     return self._get_from_file(frame_id)
                 else:
                     raise
         else:
-            raise IndexNotYetThereError
+            raise IndexError
 
     def _get_slice(self, key):
         key_range = range(*key.indices(len(self)))
@@ -267,7 +281,10 @@ class LimaClient:
 
     def _get_from_server(self, frame_id):
         if self._buffer_max_number <= (self._last_index - frame_id):
-            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+            # if frame_id is in the current acquisition, then it's not in the
+            # buffer any longer. If another acquisition started, it's not here
+            # neither.
+            raise IndexError
 
         acq_in_scan, file_in_acq, frame_in_file = self._decompose_frame_id(frame_id)
         frame_in_acq = file_in_acq * self._frames_per_file + frame_in_file
@@ -278,15 +295,24 @@ class LimaClient:
         if not self._is_acquisition_still_on_server(
             acq_in_scan + self._acquisition_offset
         ):
-            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+            raise IndexError
 
-        frame = image_utils.image_from_server(self.proxy, frame_in_acq)
-        assert frame_in_acq == frame.frame_id
+        try:
+            frame = image_utils.image_from_server(self.proxy, frame_in_acq)
+        except (RuntimeError, image_utils.NoImageAvailable) as e:
+            raise IndexError from e
+
+        if frame_in_acq != frame.frame_id:
+            _logger.warning(
+                f"Requested frame {frame_in_acq} from lima device, "
+                f"got frame {frame.frame_id}. Discarding it."
+            )
+            raise IndexError
 
         if not self._is_acquisition_still_on_server(
             acq_in_scan + self._acquisition_offset
         ):
-            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+            raise IndexError
 
         return frame.array
 
@@ -296,19 +322,13 @@ class LimaClient:
             ref.file_path, ref.data_path, ref.index, ref.format
         )
 
-    def get_references(self, key) -> ImageReference | list[ImageReference]:
-        # NOTE: _last_index is used to define when references are available.
-        # It should normally be _last_index_saved instead, but then, a LimaView
-        # would not return the same amount of images and references.
-        saved_len = self._last_index + 1
+    def get_references(self, key: int | slice) -> ImageReference | list[ImageReference]:
+        """Create references no matter the images are available or not. It is
+        caller responsibility to query indices that make sense."""
         if isinstance(key, int):
-            if key < 0:
-                key += saved_len
-            elif key >= saved_len:
-                raise IndexNotYetThereError
             return self._file_ref(key)
         elif isinstance(key, slice):
-            key_range = range(*key.indices(saved_len))
+            key_range = range(*key.indices(len(self)))
             return [self._file_ref(i) for i in key_range]
         else:
             raise TypeError(

{blissdata-2.4.0 → blissdata-2.4.2}/src/blissdata/streams/lima/stream.py

@@ -28,8 +28,8 @@ from blissdata.lima.image_utils import ImageData
 
 
 class LimaView(BaseView):
-    def __init__(self, client, start, stop):
-        self._client = client
+    def __init__(self, stream: "LimaStream", start: int, stop: int) -> None:
+        self._stream = stream
         self._view_range = range(start, stop)
 
     @property
@@ -41,13 +41,13 @@ class LimaView(BaseView):
 
     def get_data(self, start=None, stop=None):
         trimmed_range = self._view_range[start:stop]
-        return self._client[trimmed_range.start : trimmed_range.stop]
+        return self._stream[trimmed_range.start : trimmed_range.stop]
 
     def get_references(
         self, start=None, stop=None
     ) -> ImageReference | list[ImageReference]:
         trimmed_range = self._view_range[start:stop]
-        return self._client.get_references(
+        return self._stream.get_references(
             slice(trimmed_range.start, trimmed_range.stop)
         )
 
@@ -67,10 +67,16 @@ class LimaStream(BaseStream, LimaDirectAccess):
     def __init__(self, event_stream):
         super().__init__(event_stream)
         self._client = LimaClient(
-            event_stream._data_store, **event_stream.info["lima_info"]
+            data_store=event_stream._data_store,
+            request_update_cb=self._force_update_client,
+            **event_stream.info["lima_info"],
         )
         self._cursor = Stream(event_stream).cursor()
 
+    def _force_update_client(self):
+        _, status_list, sealed = self._event_stream._read_last(0)
+        self._client.update(**status_list[0], sealed=sealed)
+
     @staticmethod
     def make_definition(
         name: str,
@@ -145,31 +151,44 @@ class LimaStream(BaseStream, LimaDirectAccess):
         return tuple(self.info["shape"])
 
     def __len__(self):
-        self._update_client()
+        self._update_client_if_needed(-1)
         return len(self._client)
 
-    def _update_client(self):
+    def _update_client_if_needed(self, key: int | slice):
+        if isinstance(key, slice):
+            need_update = key.stop is None or not (0 <= key.stop <= len(self._client))
+        else:
+            need_update = not (0 <= key < len(self._client))
+        if not need_update:
+            return
+
         try:
             view = self._cursor.read(block=False, last_only=True)
         except EndOfStream:
-            return
-        if view is not None:
+            view = None
+        # use ._seal instead of .is_sealed() to skip redis query and because
+        # we just updated it by reading cursor.
+        sealed = self._event_stream._seal is not None
+        if view is None:
+            self._client.update(sealed=sealed)
+        else:
             last_status = view.get_data(-1, None)[0]
-            self._client.update(**last_status)
+            self._client.update(**last_status, sealed=sealed)
 
     def __getitem__(self, key):
-        if isinstance(key, slice):
-            need_update = key.stop is None or not (0 <= key.stop < len(self._client))
-        else:
-            need_update = not (0 <= key < len(self._client))
+        return self._get_from_client(key, reference_mode=False)
 
-        if need_update:
-            self._update_client()
+    def get_references(self, key) -> ImageReference | list[ImageReference]:
+        return self._get_from_client(key, reference_mode=True)
 
+    def _get_from_client(self, key, reference_mode):
+        self._update_client_if_needed(key)
         try:
-            return self._client[key]
+            if reference_mode:
+                return self._client.get_references(key)
+            else:
+                return self._client[key]
         except IndexError:
-            # TODO could be verified before asking the client
             index = key.start if isinstance(key, slice) else key
             if index is None:
                 index = 0
@@ -177,7 +196,7 @@ class LimaStream(BaseStream, LimaDirectAccess):
                 index += len(self._client)
 
             if index >= len(self._client):
-                if self._event_stream.is_sealed():
+                if self._client._stream_sealed:
                     raise IndexWontBeThereError
                 else:
                     raise IndexNotYetThereError
@@ -189,7 +208,7 @@ class LimaStream(BaseStream, LimaDirectAccess):
         return True
 
     def _build_view_from_events(self, index: int, events: EventRange, last_only: bool):
-        self._client.update(**events.data[-1])
+        self._client.update(**events.data[-1], sealed=events.end_of_stream)
 
         if len(self._client) <= index:
             # no new images despite client update
@@ -199,34 +218,7 @@ class LimaStream(BaseStream, LimaDirectAccess):
             start = len(self._client) - 1
         else:
             start = index
-        return LimaView(self._client, start, len(self._client))
-
-    def get_references(self, key) -> ImageReference | list[ImageReference]:
-        if isinstance(key, slice):
-            need_update = key.stop is None or not (0 <= key.stop < len(self._client))
-        else:
-            need_update = not (0 <= key < len(self._client))
-
-        if need_update:
-            self._update_client()
-
-        try:
-            return self._client.get_references(key)
-        except IndexError:
-            # TODO could be verified before asking the client
-            index = key.start if isinstance(key, slice) else key
-            if index is None:
-                index = 0
-            elif index < 0:
-                index += len(self._client)
-
-            if index >= len(self._client):
-                if self._event_stream.is_sealed():
-                    raise IndexWontBeThereError
-                else:
-                    raise IndexNotYetThereError
-            else:
-                raise IndexNoMoreThereError
+        return LimaView(self, start, len(self._client))
 
     def get_last_live_image(self) -> ImageData:
         return self._client.get_last_live_image()

blissdata-2.4.2/src/blissdata/tests/streams/plugins/test_lima_stream.py

@@ -0,0 +1,173 @@
+import pytest
+import time
+from threading import Thread
+from contextlib import contextmanager
+
+from blissdata.lima.client import LimaClient
+from blissdata.streams.encoding.json import JsonStreamEncoder
+from blissdata.streams.event_stream import EventStream
+from blissdata.streams.lima import LimaStream
+from blissdata.exceptions import (
+    IndexNoMoreThereError,
+    IndexNotYetThereError,
+    IndexWontBeThereError,
+)
+
+
+class LimaDataSimulator:
+    def __init__(self, rw_stream, buffer_size, acq_size, file_size=None):
+        self.rw_stream = rw_stream
+        self.last_index = -1
+        self.last_index_saved = -1
+
+        self.buffer_size = buffer_size
+        self.acq_size = acq_size
+        self.file_size = file_size
+
+    def publish_images(self, last_index=-1, last_index_saved=-1):
+        self.last_index = max(self.last_index, last_index)
+        self.last_index_saved = max(self.last_index_saved, last_index_saved)
+        self.rw_stream.send(
+            {"last_index": self.last_index, "last_index_saved": self.last_index_saved}
+        )
+        self.rw_stream.join()
+
+    def get_from_server_mock(self, frame_id: int):
+        if frame_id > self.last_index:
+            raise IndexError("Not acquired yet")
+        if frame_id <= self.last_index - self.buffer_size:
+            raise IndexError("Buffer expired")
+        print(f"read {frame_id} from memory")
+        return f"memo-{frame_id}"
+
+    def get_from_file_mock(self, frame_id):
+        if self.file_size is None:
+            raise Exception("Stream is not being saved")
+        if frame_id > (self.last_index_saved // self.file_size * self.file_size):
+            raise Exception("File not complete")
+        print(f"read {frame_id} from file")
+        return f"file-{frame_id}"
+
+
+@contextmanager
+def lima_stream_pair(data_store, buffer_size, acq_size, file_size=None):
+    info = {
+        "protocol_version": LimaClient.PROTOCOL_VERSION,
+        "server_url": "cafe",
+        "buffer_max_number": buffer_size,
+        "frame_per_acquisition": acq_size,
+        "acquisition_offset": 0,
+    }
+    if file_size is not None:
+        info.update(
+            {
+                "file_offset": 0,
+                "frame_per_file": file_size,
+                "file_path": "foo",
+                "data_path": "bar",
+                "file_format": "hdf5",
+            }
+        )
+
+    encoder = JsonStreamEncoder()
+    model = data_store._stream_model(encoding=encoder.info(), info={"lima_info": info})
+    event_rw_stream = EventStream.create(data_store, "test_stream", model)
+    event_ro_stream = EventStream.open(data_store, "test_stream", model)
+
+    try:
+        yield LimaStream(event_rw_stream), LimaStream(event_ro_stream)
+    finally:
+        event_rw_stream._sink.stop()
+
+
+def test_get_image_no_file(monkeypatch, data_store):
+    params = dict(buffer_size=10, acq_size=1000, file_size=None)
+
+    with lima_stream_pair(data_store, **params) as (rw_stream, ro_stream):
+        data_simu = LimaDataSimulator(rw_stream, **params)
+        monkeypatch.setattr(
+            LimaClient, "_get_from_server", data_simu.get_from_server_mock
+        )
+        monkeypatch.setattr(LimaClient, "_get_from_file", data_simu.get_from_file_mock)
+
+        # not image at all
+        assert len(ro_stream) == 0
+        with pytest.raises(IndexNotYetThereError):
+            ro_stream[0]
+
+        # 0 to 4
+        data_simu.publish_images(last_index=4)
+        assert len(ro_stream) == 5
+        assert ro_stream[0] == "memo-0"
+        assert ro_stream[4] == "memo-4"
+        with pytest.raises(IndexNotYetThereError):
+            ro_stream[5]
+
+        # 5 to 14 (buffer wrapped around)
+        data_simu.publish_images(last_index=14)
+        assert len(ro_stream) == 15
+        assert ro_stream[5] == "memo-5"
+        assert ro_stream[14] == "memo-14"
+        with pytest.raises(IndexNoMoreThereError):
+            ro_stream[4]
+        with pytest.raises(IndexNotYetThereError):
+            ro_stream[15]
+
+        rw_stream.seal()
+        with pytest.raises(IndexWontBeThereError):
+            ro_stream[15]
+
+
+@pytest.mark.timeout(60)
+def test_get_image_file(monkeypatch, data_store):
+    params = dict(buffer_size=10, acq_size=1000, file_size=20)
+
+    with lima_stream_pair(data_store, **params) as (rw_stream, ro_stream):
+        data_simu = LimaDataSimulator(rw_stream, **params)
+        monkeypatch.setattr(
+            LimaClient, "_get_from_server", data_simu.get_from_server_mock
+        )
+        monkeypatch.setattr(LimaClient, "_get_from_file", data_simu.get_from_file_mock)
+
+        # buffer [0, 9], file [None]
+        data_simu.publish_images(last_index=9, last_index_saved=9)
+        assert len(ro_stream) == 10
+        assert ro_stream[0] == "memo-0"
+        assert ro_stream[9] == "memo-9"
+        with pytest.raises(IndexNotYetThereError):
+            ro_stream[10]
+
+        # buffer [15, 24], file [0-19]
+        data_simu.publish_images(last_index=24, last_index_saved=20)
+        assert len(ro_stream) == 25
+        assert ro_stream[0] == "file-0"
+        assert ro_stream[9] == "file-9"
+        assert ro_stream[19] == "file-19"
+        assert ro_stream[20] == "memo-20"
+        assert ro_stream[24] == "memo-24"
+        with pytest.raises(IndexNotYetThereError):
+            ro_stream[25]
+
+        # buffer [25, 34], file [0-19] ----> dead zone [20, 24]
+        data_simu.publish_images(last_index=34, last_index_saved=20)
+
+        def complete_file_soon():
+            time.sleep(0.5)
+            # buffer [31, 40], file [0-39] ----> file caught up the dead zone
+            data_simu.publish_images(last_index=40, last_index_saved=40)
+
+        t = Thread(target=complete_file_soon)
+        t.start()
+        # reading in the dead zone will wait for file index to catch up
+        assert ro_stream[24] == "file-24"
+        t.join()
+
+        # read from memory if file is not complete
+        assert ro_stream[40] == "memo-40"
+
+        rw_stream.seal()
+        with pytest.raises(IndexWontBeThereError):
+            ro_stream[41]
+
+        # read from file after stream sealing
+        assert ro_stream[40] == "file-40"

{blissdata-2.4.0 → blissdata-2.4.2/src/blissdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blissdata
-Version: 2.4.0
+Version: 2.4.2
 Summary: Bliss data streaming client
 Maintainer: BCU (ESRF)
 License-Expression: LGPL-3.0-or-later

{blissdata-2.4.0 → blissdata-2.4.2}/src/blissdata.egg-info/SOURCES.txt

@@ -95,4 +95,5 @@ src/blissdata/tests/streams/encoding/test_json.py
 src/blissdata/tests/streams/encoding/test_numeric.py
 src/blissdata/tests/streams/plugins/__init__.py
 src/blissdata/tests/streams/plugins/test_file_backed_stream.py
+src/blissdata/tests/streams/plugins/test_lima_stream.py
 src/blissdata/tests/streams/plugins/test_plugin_base.py