blissdata 0.3.2__tar.gz → 1.0.0__tar.gz

blissdata-1.0.0/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: blissdata
+Version: 1.0.0
+Summary: Bliss data streaming client
+Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blissdata
+Author: BCU (ESRF)
+License: LGPLv3
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
+Classifier: Programming Language :: Python :: 3
+Requires-Python: <3.10,>=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beartype
+Requires-Dist: h5py<3.10,>=2.9
+Requires-Dist: msgpack
+Requires-Dist: numpy>=1.13
+Requires-Dist: redis>=4.6.0
+Requires-Dist: redis-om
+Requires-Dist: silx>=1.1.0
+Provides-Extra: tango
+Requires-Dist: pytango>=9.3.1; extra == "tango"
+Provides-Extra: test
+Requires-Dist: gevent>=20.9.0; extra == "test"
+Requires-Dist: pytest>=7; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-redis; extra == "test"
+Requires-Dist: pytest-timeout; extra == "test"
+Provides-Extra: dev
+Requires-Dist: gevent>=20.9.0; extra == "dev"
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: pytest-redis; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
+Requires-Dist: black==22.3; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
+Provides-Extra: beacon
+Requires-Dist: ruamel.yaml; extra == "beacon"
+
+# Blissdata
+
+Documentation [here](https://bliss.gitlab-pages.esrf.fr/bliss/master/blissdata/getting_started.html).

blissdata-1.0.0/README.md

@@ -0,0 +1,3 @@
+# Blissdata
+
+Documentation [here](https://bliss.gitlab-pages.esrf.fr/bliss/master/blissdata/getting_started.html).

{blissdata-0.3.2/blissdata/data/nodes → blissdata-1.0.0/blissdata}/__init__.py

@@ -4,8 +4,4 @@
 #
 # Copyright (c) 2015-2023 Beamline Control Unit, ESRF
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
-
-"""Data management
-
-   node data types that map to structure in redis
-"""
+__version__ = "1.0.0"

{blissdata-0.3.2 → blissdata-1.0.0}/blissdata/beacon/_base.py

@@ -1,5 +1,6 @@
 """Base client to communicate with Beacon."""
 
+import platform
 import socket
 import struct
 import threading
@@ -28,7 +29,8 @@ class BeaconClient:
         else:
             self._address = host, port
         connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
+        if platform.system() != "Windows":
+            connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
         connection.connect(self._address)
         connection.settimeout(timeout)
         self._connection = connection

{blissdata-0.3.2 → blissdata-1.0.0}/blissdata/beacon/data.py

@@ -16,7 +16,8 @@ class BeaconData(BeaconClient):
     REDIS_DATA_SERVER_OK = 34
 
     def get_redis_db(self) -> str:
-        """Returns the URL of the Redis database that contains the Bliss settings. For example 'foobar:25001'."""
+        """Returns the URL of the Redis database that contains the Bliss settings.
+        For example 'redis://foobar:25001' or 'unix:///tmp/redis.sock'."""
         msg = b"%s%s" % (struct.pack("<ii", self.REDIS_QUERY, 0), b"")
         self._connection.sendall(msg)
         data = b""
@@ -32,14 +33,23 @@ class BeaconData(BeaconClient):
             break
         if message_type != self.REDIS_QUERY_ANSWER:
             raise RuntimeError(f"Unexpected message type '{message_type}'")
-        return message.decode()
+        host, port = message.decode().split(":")
+        if host == "localhost":
+            return f"unix://{port}"
+        else:
+            return f"redis://{host}:{port}"
 
     def get_redis_data_db(self) -> str:
-        """Returns the URL of the Redis database that contains the Bliss scan data. For example 'foobar:25002'."""
+        """Returns the URL of the Redis database that contains the Bliss scan data.
+        For example 'redis://foobar:25002' or 'unix:///tmp/redis_data.sock'."""
         response = self._request(self.REDIS_DATA_SERVER_QUERY, "")
         response_type, data = response.read()
         if response_type == self.REDIS_DATA_SERVER_OK:
-            return data.decode().replace("|", ":", 1)
+            host, port = data.decode().split("|")[:2]
+            if host == "localhost":
+                return f"unix://{port}"
+            else:
+                return f"redis://{host}:{port}"
         elif response_type == self.REDIS_DATA_SERVER_FAILED:
             raise RuntimeError(data.decode())
         raise RuntimeError(f"Unexpected Beacon response type {response_type}")

{blissdata-0.3.2 → blissdata-1.0.0}/blissdata/h5api/dynamic_hdf5.py

@@ -74,7 +74,10 @@ class Group(Node, abstract.Group):
 
     def __getitem__(self, key: str) -> Node:
         if not key.startswith(hdf5.SEP):
-            key = f"{self.name}{hdf5.SEP}{key}"
+            if self.name == hdf5.SEP:
+                key = f"{hdf5.SEP}{key}"
+            else:
+                key = f"{self.name}{hdf5.SEP}{key}"
         h5py_obj = self._retry_handler.get_item(key)
         if isinstance(h5py_obj, h5py.Group):
             return Group(self._retry_handler, h5py_obj.name)

{blissdata-0.3.2 → blissdata-1.0.0}/blissdata/h5api/utils/hdf5.py

@@ -10,7 +10,7 @@ from . import types
 from . import lima
 from .hdf5_retry import retry_file_access
 from .hdf5_retry import ignore_retry_timeout
-from .hdf5_retry import RetryError, SoftRetryError
+from .hdf5_retry import RetryTimeoutError, RetryError, SoftRetryError
 
 
 HDF5Item = Union[h5py.Dataset, h5py.Group, h5py.File, lima.LimaDataset, lima.LimaGroup]
@@ -71,13 +71,19 @@ class DynamicHDF5Handler:
         return self._file_obj
 
     def get_item(self, name: str) -> HDF5Item:
-        return self._retry_get_item(name)
+        try:
+            return self._retry_get_item(name)
+        except RetryTimeoutError as e:
+            raise KeyError(name) from e
 
     def slice_dataset(self, name: str, idx: types.DataIndexType) -> types.DataType:
         return self._retry_slice_dataset(name, idx)
 
     def get_attr(self, name: str, key: str) -> Any:
-        return self._retry_get_attr(name, key)
+        try:
+            return self._retry_get_attr(name, key)
+        except RetryTimeoutError as e:
+            raise KeyError(name) from e
 
     @ignore_retry_timeout
     def iter_attrs(self, name: str) -> Iterator[str]:
@@ -138,7 +144,21 @@ class DynamicHDF5Handler:
         item = self._native_items.get(name)
         if item is not None:
             return item
-        item = self.file_obj[name]
+        try:
+            item = self.file_obj[name]
+        except KeyError as e:
+            parent_name = name.rpartition(SEP)[0]
+            if not parent_name:
+                parent_name = SEP
+            if parent_name != name:
+                try:
+                    parent_item = self._get_item(parent_name)
+                except KeyError:
+                    pass
+                else:
+                    if self._is_finished(parent_item):
+                        raise RetryTimeoutError(str(e)) from e
+            raise
         self._native_items[name] = item
         return item
 
@@ -152,7 +172,12 @@ class DynamicHDF5Handler:
 
     def _get_attr(self, name: str, key: str) -> Any:
         item = self._get_item(name)
-        return item.attrs[key]
+        try:
+            return item.attrs[key]
+        except KeyError as e:
+            if self._is_finished(item):
+                raise RetryTimeoutError(str(e)) from e
+            raise
 
     def _iter_attrs(self, name: str, start_index: int = 0) -> Iterator[str]:
         item = self._get_item(name)

{blissdata-0.3.2 → blissdata-1.0.0}/blissdata/h5api/utils/hdf5_retry.py

@@ -22,7 +22,7 @@ def ignore_retry_timeout(method: Callable) -> Callable:
         @wraps(method)
         def wrapper(self, *args, **kw):
             try:
-                yield from method(self, *args, *kw)
+                yield from method(self, *args, **kw)
             except RetryTimeoutError:
                 pass
 

blissdata-1.0.0/blissdata/lima/client.py

@@ -0,0 +1,315 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from __future__ import annotations
+import math
+import numpy as np
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional
+
+from blissdata.redis_engine.exceptions import (
+    IndexNoMoreThereError,
+    IndexNotYetThereError,
+)
+from . import image_utils
+
+try:
+    from gevent.monkey import is_anything_patched
+except ImportError:
+    use_gevent = False
+else:
+    use_gevent = is_anything_patched()
+
+try:
+    if use_gevent:
+        from tango.gevent import DeviceProxy
+    else:
+        from tango import DeviceProxy
+    from tango import DevSource
+except ImportError:
+    DeviceProxy = None
+    DevSource = None
+
+
+def acquisition_on_server(data_store, server_url):
+    return int(data_store._redis.get(server_url).decode())
+
+
+def prepare_next_lima_acquisition(data_store, server_url):
+    pipe = data_store._redis.pipeline(transaction=True)
+    # when the key doesn't exist, set it to -1 before .incr()
+    pipe.setnx(server_url, -1)
+    pipe.incr(server_url)
+    return pipe.execute()[1]
+
+
+@dataclass
+class ImageReference:
+    format: str
+    file_path: str
+    data_path: Optional[str]
+    index: int
+
+
+class LimaClientInterface(ABC):
+    """
+    Interface which have to be used to provide a service to LimaStream
+    """
+
+    @abstractmethod
+    def update(self, *args, **kwargs):
+        raise NotImplementedError
+
+    @abstractmethod
+    def __len__(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def __getitem__(self, frame_id):
+        raise NotImplementedError
+
+
+def lima_client_factory(
+    data_store, channel_info: dict, ref_mode: bool = False
+) -> LimaClientInterface:
+    """Factory to instanciate the appropriate lima client (Lima1, Lima2) in
+    either normal mode (client returns images) or ref mode (client returns
+    references to files)."""
+    format = channel_info.get("format")
+    if format == "lima_v1":
+        if ref_mode:
+            return LimaRefClient(data_store, **channel_info["lima_info"])
+        else:
+            return LimaClient(data_store, **channel_info["lima_info"])
+    elif format == "lima_v2":
+        raise NotImplementedError("No client for Lima2 yet")
+    else:
+        raise ValueError(f"Not a lima stream, format: {format}")
+
+
+class LimaClient(LimaClientInterface):
+    PROTOCOL_VERSION = 1
+
+    def __init__(self, data_store, **lima_info):
+        protocol = lima_info["protocol_version"]
+        if self.PROTOCOL_VERSION != protocol:
+            raise Exception(
+                f"{type(self).__name__} supports lima json protocol {self.PROTOCOL_VERSION}, found version {protocol}"
+            )
+
+        self._data_store = data_store
+        self._proxy = None
+        self._server_url = lima_info["server_url"]
+        self._buffer_max_number = lima_info["buffer_max_number"]
+        self._frame_per_acquisition = lima_info["frame_per_acquisition"]
+        self._acquisition_offset = lima_info["acquisition_offset"]
+        self._acq_trigger_mode: str | None = None
+
+        self._saved = "file_path" in lima_info
+        if self._saved:
+            self._file_offset = lima_info["file_offset"]
+            self._frame_per_file = lima_info["frame_per_file"]
+            self._file_path = lima_info["file_path"]
+            self._data_path = lima_info["data_path"]
+            self._file_format = lima_info["file_format"]
+            self._file_per_acquisition = math.ceil(
+                self._frame_per_acquisition / self._frame_per_file
+            )
+        else:
+            self._file_format = None
+            self._frame_per_file = 0
+
+        self._last_index = -1
+        self._last_index_saved = -1
+        self._last_index_saved_and_closed = -1
+        self._last_acq_seen_on_server = -1
+
+    @property
+    def acq_trigger_mode(self) -> str:
+        """Returns the cached acq trigger mode of this detector"""
+        if self._acq_trigger_mode is not None:
+            return self._acq_trigger_mode
+        acq_trigger_mode = self.proxy.acq_trigger_mode
+        self._acq_trigger_mode = acq_trigger_mode
+        return acq_trigger_mode
+
+    @property
+    def file_format(self):
+        return self._file_format
+
+    def __len__(self):
+        return self._last_index + 1
+
+    def update(self, last_index, last_index_saved):
+        self._last_index = last_index
+        self._last_index_saved = last_index_saved
+        if self._saved:
+            self._last_index_saved_and_closed = self._last_readable_index(
+                last_index_saved
+            )
+
+    def _decompose_frame_id(self, frame_id):
+        try:
+            acq_in_scan, frame_in_acq = divmod(frame_id, self._frame_per_acquisition)
+        except ZeroDivisionError:
+            acq_in_scan, frame_in_acq = 0, frame_id
+        try:
+            file_in_acq, frame_in_file = divmod(frame_in_acq, self._frame_per_file)
+        except ZeroDivisionError:
+            file_in_acq, frame_in_file = 0, frame_in_acq
+        return acq_in_scan, file_in_acq, frame_in_file
+
+    def _last_readable_index(self, frame_id):
+        """Return the last index that can be read from a file (i.e. file is complete)"""
+        acq_in_scan, file_in_acq, _ = self._decompose_frame_id(frame_id)
+        return (
+            acq_in_scan * self._file_per_acquisition
+            + file_in_acq * self._frame_per_file
+            - 1
+        )
+
+    def _is_acquisition_still_on_server(self, acq_id):
+        if acq_id < self._last_acq_seen_on_server:
+            # We already know this acquisition is outdated
+            return False
+        self._last_acq_seen_on_server = acquisition_on_server(
+            self._data_store, self._server_url
+        )
+        return acq_id == self._last_acq_seen_on_server
+
+    @property
+    def proxy(self):
+        """Lazy connection to the tango device, because a lima client may never use it."""
+        if self._proxy is None:
+            if DeviceProxy is None:
+                raise RuntimeError("requires 'pytango' to be installed")
+            self._proxy = DeviceProxy(self._server_url)
+            self._proxy.set_source(DevSource.DEV)
+        return self._proxy
+
+    def get_last_live_image(self) -> image_utils.ImageData:
+        """Returns the last frame from the memory buffer of Lima.
+
+        This code does not check if the frame is part of the actual scan.
+
+        Raises:
+            NoImageAvailable: when the lima server buffer does not yet contain any frame
+            ImageFormatNotSupported: when the retrieved data is not supported
+        """
+        data = image_utils.image_from_server(self.proxy, -1)
+
+        def normalize_frame_id(frame_id: int | None) -> int | None:
+            if frame_id == 0:
+                trigger_mode = self.acq_trigger_mode
+                if trigger_mode == "INTERNAL_TRIGGER":
+                    # Because of the bliss/lima architecture we can't really know the index
+                    if self._last_index == 0:
+                        # For a ct the index is valid
+                        return 0
+                    return None
+            return frame_id
+
+        frame_id = normalize_frame_id(data.frame_id)
+        return image_utils.ImageData(data.array, frame_id)
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self._get_index(key)
+        elif isinstance(key, slice):
+            return self._get_slice(key)
+        else:
+            raise TypeError(
+                f"{type(self).__name__} indices must be integers or slices, not {type(key).__name__}"
+            )
+
+    def _get_index(self, frame_id):
+        if frame_id == -1:
+            return self.get_last_live_image().array
+        if frame_id < -1:
+            raise NotImplementedError("Negative index not supported")
+
+        if self._saved and frame_id <= self._last_index_saved_and_closed:
+            # immediately accessible from file
+            return self._get_from_file(frame_id)
+        elif frame_id <= self._last_index:
+            # try to access on lima server
+            try:
+                return self._get_from_server(frame_id)
+            except IndexNoMoreThereError:
+                # wait for image to become available from file
+                # TODO wait_for_file(timeout)
+                if self._saved:
+                    return self._get_from_file(frame_id)
+                else:
+                    raise
+        else:
+            raise IndexNotYetThereError
+
+    def _get_slice(self, key):
+        key_range = range(*key.indices(len(self)))
+        return np.array([self._get_index(i) for i in key_range])
+
+    def _file_ref(self, frame_id) -> ImageReference:
+        """Convert a frame_id to an image reference to access from file.
+        IMPORTANT: It doesn't mean such frame_id is actually available."""
+        acq_in_scan, file_in_acq, frame_in_file = self._decompose_frame_id(frame_id)
+        file_in_scan = acq_in_scan * self._file_per_acquisition + file_in_acq
+        file_number = file_in_scan + self._file_offset
+        file_path = self._file_path % (file_number)
+        return ImageReference(
+            self._file_format, file_path, self._data_path, frame_in_file
+        )
+
+    def _get_from_server(self, frame_id):
+        if self._buffer_max_number <= (self._last_index - frame_id):
+            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+
+        acq_in_scan, file_in_acq, frame_in_file = self._decompose_frame_id(frame_id)
+        frame_in_acq = file_in_acq * self._frame_per_file + frame_in_file
+
+        # We need to check the server acquisition is the same before AND after reading the image.
+        # This is the only way to ensure the image belong to an acquisition. This is due to the
+        # lack of atomic operation to get an image and the current acquisition on server.
+        if not self._is_acquisition_still_on_server(
+            acq_in_scan + self._acquisition_offset
+        ):
+            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+
+        image, index = image_utils.image_from_server(self.proxy, frame_in_acq)
+        assert frame_in_acq == index
+
+        if not self._is_acquisition_still_on_server(
+            acq_in_scan + self._acquisition_offset
+        ):
+            raise IndexNoMoreThereError()  # TODO IndexError here and more details in LimaStream ???
+
+        return image
+
+    def _get_from_file(self, frame_id):
+        ref = self._file_ref(frame_id)
+        return image_utils.image_from_file(
+            ref.file_path, ref.data_path, ref.index, ref.format
+        )
+
+
+class LimaRefClient(LimaClient):
+    def __getitem__(self, key):
+        saved_len = self._last_index_saved + 1
+        if isinstance(key, int):
+            if key < 0:
+                key += saved_len
+            elif key >= saved_len:
+                raise IndexNotYetThereError
+            return self._file_ref(key)
+        elif isinstance(key, slice):
+            key_range = range(*key.indices(saved_len))
+            return [self._file_ref(i) for i in key_range]
+        else:
+            raise TypeError(
+                f"{type(self).__name__} indices must be integers or slices, not {type(key).__name__}"
+            )