lima2.client 2.0.2__tar.gz → 2.1.0__tar.gz

{lima2_client-2.0.2/src/lima2.client.egg-info → lima2_client-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lima2.client
-Version: 2.0.2
+Version: 2.1.0
 Summary: Python client library for Lima2 detector
 Home-page: https://gitlab.esrf.fr/limagroup/lima2-client
 Author: BCU (ESRF)
@@ -19,6 +19,7 @@ Requires-Dist: beartype
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: typing-extensions
+Requires-Dist: h5py
 Provides-Extra: shell
 Requires-Dist: ipython; extra == "shell"
 

{lima2_client-2.0.2 → lima2_client-2.1.0}/setup.cfg

@@ -22,6 +22,7 @@ install_requires =
 	numpy
 	pyyaml
 	typing-extensions
+	h5py
 
 [options.entry_points]
 console_scripts = 
@@ -40,7 +41,7 @@ per-file-ignores =
 required_plugins = 
 	pytest-cov>=6.0.0
 	pytest-forked>=1.6.0
-addopts = -v --forked
+addopts = -v
 pythonpath = src
 
 [coverage:run]

{lima2_client-2.0.2 → lima2_client-2.1.0}/src/lima2/client/client.py

@@ -18,7 +18,10 @@ individual receivers, for instance if one of them is stuck on a frame while the
 processing data.
 """
 
+import logging
 import os
+import traceback
+from concurrent import futures
 from uuid import UUID
 
 import tango as tg
@@ -33,6 +36,8 @@ from lima2.client.progress_counter import ProgressCounter, SingleCounter
 from lima2.client.topology import TopologyKind
 from lima2.client.utils import docstring_from
 
+logger = logging.getLogger(__name__)
+
 
 @beartype
 class Client:
@@ -67,12 +72,10 @@ class Client:
                 f"No matching topology type for {repr(topology_name)}"
             ) from e
 
-        self.recv_devs = rcv_devs
-
         # Cache of processing pipelines by uuid (see method `pipeline`)
         self.__pipelines: dict[UUID, Pipeline] = {}
 
-        self.tango_db = tg.Database()
+        self._tango_db = tg.Database()
 
     @classmethod
     def from_yaml(
@@ -91,6 +94,16 @@ class Client:
 
         return Client(ctl_dev=ctl_dev, rcv_devs=rcv_devs)
 
+    @property
+    def control(self) -> tg.DeviceProxy:
+        """Lima2 control tango device."""
+        return self.detector.ctrl
+
+    @property
+    def receivers(self) -> list[tg.DeviceProxy]:
+        """Lima2 receiver tango devices."""
+        return self.detector.recvs
+
     @property
     def detector(self) -> Detector:
         return self._detector
@@ -124,6 +137,11 @@ class Client:
     def params_default(self) -> dict[str, Any]:
         return self.detector.params_default
 
+    @property
+    @docstring_from(Detector.params_schema)
+    def params_schema(self) -> dict[str, Any]:
+        return self.detector.params_schema
+
     ####################################################################################
     # Control
     ####################################################################################
@@ -138,7 +156,7 @@ class Client:
             ctl_params=ctrl_params,
             acq_params=acq_params,
             proc_params=proc_params,
-            num_receivers=len(self.recv_devs),
+            num_receivers=len(self.receivers),
         )
 
         self.detector.prepare_acq(uuid, ctl, acq, proc)
@@ -170,7 +188,7 @@ class Client:
             SingleCounter(
                 name="nb_frames_acquired",
                 value=self.detector.nb_frames_acquired,
-                source=self.detector.ctrl.name(),
+                source=self.control.name(),
             )
         )
 
@@ -184,7 +202,7 @@ class Client:
                     value=recv.nb_frames_xferred,
                     source=recv.name(),
                 )
-                for recv in self.detector.recvs
+                for recv in self.receivers
             ]
         )
 
@@ -195,7 +213,7 @@ class Client:
     @property
     def pipelines(self) -> list[UUID]:
         """Get list of available pipelines UUID"""
-        uuids = [recv.pipelines for recv in self.recv_devs]
+        uuids = [recv.pipelines for recv in self.receivers]
         if all(uuid is None for uuid in uuids):
             return []
         return [UUID(u) for u in {y for x in uuids for y in x}]
@@ -209,12 +227,13 @@ class Client:
             return self.__pipelines[uuid]
 
         # uuid isn't in the local cache of pipelines: get it from Tango
-        instances = self.tango_db.get_device_exported(f"*/limaprocessing/{uuid}*")
+        instances = self._tango_db.get_device_exported(f"*/limaprocessing/{uuid}*")
         if not instances:
             raise ValueError(f"Pipeline not found in tango database: {uuid=}")
 
         class_names = [
-            self.tango_db.get_device_info(instance).class_name for instance in instances
+            self._tango_db.get_device_info(instance).class_name
+            for instance in instances
         ]
 
         # Select the pipeline according to the Tango class (Homogeneous processing for now)
@@ -237,7 +256,7 @@ class Client:
         Raises if no pipeline has been created, or if receivers disagree on the
         current pipeline's uuid.
         """
-        uuids: list[str] = [recv.current_pipeline for recv in self.recv_devs]
+        uuids: list[str] = [recv.current_pipeline for recv in self.receivers]
 
         if all([uuid == "" for uuid in uuids]):
             raise ValueError("No pipeline present in the tango database")
@@ -250,10 +269,54 @@ class Client:
         return self.pipeline(uuids[0])
 
     def erase_pipeline(self, uuid: UUID) -> None:
-        """Erase a pipeline instance"""
-        self.__pipelines.pop(uuid, None)
-        for recv in self.recv_devs:
+        """Erase a pipeline instance.
+
+        Raises if the pipeline is absent from a receiver.
+        """
+        for recv in self.receivers:
             recv.erasePipeline(str(uuid))
+        self.__pipelines.pop(uuid, None)
+
+    def clear_previous_pipelines(self) -> None:
+        """Erase all pipelines except the current one."""
+        try:
+            current = self.current_pipeline.uuid
+        except ValueError:
+            current = None
+
+        def clear(receiver: tg.DeviceProxy):
+            """Clear previous pipelines from a single receiver device."""
+            if receiver.pipelines is None:
+                # No pipelines
+                return []
+
+            cleared = []
+            for uuid in receiver.pipelines:
+                if uuid != str(current):
+                    logger.debug(
+                        f"Erasing pipeline {uuid} from recv {receiver.dev_name()}"
+                    )
+                    receiver.erasePipeline(uuid)
+                    cleared.append(uuid)
+            return cleared
+
+        cleared: set[str] = set()
+        with futures.ThreadPoolExecutor(max_workers=len(self.receivers)) as executor:
+            fs = [executor.submit(clear, receiver) for receiver in self.receivers]
+            for receiver, future in zip(self.receivers, fs):
+                try:
+                    cleared = cleared.union(future.result())
+                except Exception:
+                    logger.warning(
+                        f"Exception while clearing pipelines from {receiver.dev_name()}: "
+                        f"{traceback.format_exc()}"
+                    )
+
+        logger.debug(f"Cleared pipelines {[uuid for uuid in set(cleared)]}")
+
+        # Reset local cache
+        if current is not None:
+            self.__pipelines = {current: self.__pipelines[current]}
 
     ####################################################################################
     # Utility
@@ -264,10 +327,10 @@ class Client:
             [
                 "Lima2 client",
                 f"State: {self.detector.state}",
-                "Controller:",
-                f" {self.detector.ctrl}",
+                "Control:",
+                f" {self.control}",
                 f"Receivers ({self.topology_kind}):",
-                *[f" - {recv}" for recv in self.detector.recvs],
+                *[f" - {recv}" for recv in self.receivers],
                 "Progress counters:",
                 f" - {self.nb_frames_acquired}",
                 f" - {self.nb_frames_xferred}",

lima2_client-2.1.0/src/lima2/client/convert.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the Lima2 project
+#
+# Copyright (c) 2020-2024 Beamline Control Unit, ESRF
+# Distributed under the MIT license. See LICENSE for more info.
+
+"""Convertion routines.
+
+Frame info is serialized as {'dimensions': {'x': 2068, 'y': 512}, 'nb_channels': 2, 'pixel_type': 'gray16'}
+"""
+
+import numpy as np
+
+# convert from pixel enum type to numpy type
+pixel_type_to_np_dtype = {
+    "gray8s": np.int8,
+    "gray8": np.uint8,
+    "gray16s": np.int16,
+    "gray16": np.uint16,
+    "gray32s": np.int32,
+    "gray32": np.uint32,
+    "gray32f": np.float32,
+    "gray64f": np.float64,
+}
+
+
+def frame_info_to_shape_dtype(frame_info):
+    return dict(
+        shape=(
+            frame_info["nb_channels"],
+            frame_info["dimensions"]["y"],
+            frame_info["dimensions"]["x"],
+        ),
+        dtype=pixel_type_to_np_dtype[frame_info["pixel_type"]],
+    )

{lima2_client-2.0.2 → lima2_client-2.1.0}/src/lima2/client/detector.py

@@ -24,13 +24,15 @@ import json
 import logging
 import uuid
 
-import jsonschema
 import tango
 from beartype import beartype
 from jsonschema_default import create_from
+from typing_extensions import Optional
 
+from lima2.client import pipelines
 from lima2.client import state_machine as fsm
 from lima2.client.state_machine import State
+from lima2.client.utils import validate
 
 # Create a logger
 _logger = logging.getLogger(__name__)
@@ -45,7 +47,7 @@ class CommError(RuntimeError):
 class Detector:
     """The main Detector interface."""
 
-    DEFAULT_TANGO_TIMEOUT = 20 * 60  #
+    DEFAULT_TANGO_TIMEOUT = 5
 
     def __init__(
         self,
@@ -72,12 +74,7 @@ class Detector:
         self.__ctrl = ctrl_dev
         self.__recvs = list(recv_devs)
 
-        try:
-            self.ping()
-        except tango.DevFailed as e:
-            raise CommError(
-                "Failed to ping Tango servers (are the devices running?)"
-            ) from e
+        self.ping()
 
         rank_order = [d.recv_rank for d in recv_devs]
         if not all([r == i for i, r in enumerate(rank_order)]):
@@ -85,7 +82,7 @@ class Detector:
                 f"Receiver MPI rank order does not match client order: {rank_order}"
             )
 
-        self.__params_schema = None
+        self.__params_schema: Optional[dict] = None
         self.__fsm = fsm.StateMachine(self.__ctrl, self.__recvs)
         self.__try_sync_hard()
 
@@ -100,8 +97,6 @@ class Detector:
             d.set_green_mode(tango.GreenMode.Gevent)
             d.set_timeout_millis(timeout * 1000)
 
-        self.__try_params_schema()
-
     @property
     def det_info(self) -> dict:
         """
@@ -166,26 +161,19 @@ class Detector:
         """
         return json.loads(self.__ctrl.det_capabilities)  # type: ignore
 
-    def __try_params_schema(self):
-        """
-        Try to init the parameters from a lazy constructed device Proxy
-        (to be used when the DS is not running when Detector is constructed)
-        """
-        try:
-            if not self.__params_schema:
-                self.__params_schema = self.__get_params_schema()
-            return True
-        except (tango.DevFailed, RuntimeError) as e:
-            print(f"Failed to get params schema: {e}")
-            return False
-
     def __get_params_schema(self) -> dict:
         """
         Get the parameters's schema for each devices from the Tango class attribute
+
+        Raises tango.DevFailed if we cannot connect to database at TANGO_HOST.
+
+        If the lima2 server hasn't been started yet, returns a dict of the form:
+        {
+            d.dev_name(): {"init_params": None, "acq_params": None}
+            for d in self.devs
+        }
         """
         tango_db = tango.Database()
-        if tango_db is None:
-            return {}
 
         res = {}
         for d in self.devs:
@@ -207,7 +195,7 @@ class Detector:
         return res
 
     @property
-    def params_schema(self):
+    def params_schema(self) -> dict:
         """Returns JSON schemas of init, acquisition and processing parameters.
 
         Example:
@@ -235,10 +223,9 @@ class Detector:
         Group:
             Parameters
         """
-        if self.__params_schema:
-            return self.__params_schema
+        if self.__params_schema is None:
+            self.__params_schema = self.__get_params_schema()
 
-        self.__try_params_schema()
         return self.__params_schema
 
     @property
@@ -281,13 +268,16 @@ class Detector:
         Ping all the devices of the system.
 
         Raises:
-            tango.ConnectionFailed: if the connection failed.
+            CommError if the connection failed.
 
         Group:
             Status
         """
         for d in self.devs:
-            d.ping()
+            try:
+                d.ping()
+            except tango.DevFailed as e:
+                raise CommError(f"Failed to ping device {d.dev_name()}") from e
 
     @property
     def devs(self) -> list[tango.DeviceProxy]:
@@ -347,18 +337,11 @@ class Detector:
         Group:
             State
         """
-        states = [d.acq_state for d in self.devs]
-        if not all(s == states[0] for s in states):
-            _logger.warning(f"Inconsistant server states {states}")
-        server_state = states[0]
-        ServerState = type(server_state)
-        map = {
-            ServerState.idle: State.IDLE,
-            ServerState.prepared: State.PREPARED,
-            ServerState.running: State.RUNNING,
-            ServerState.stopped: State.IDLE,
-        }
-        self.__fsm.state = map[server_state]
+        try:
+            self.__fsm.sync()
+        except Exception as e:
+            _logger.warning(f"Unable to sync with server: {e}")
+
         return self.__fsm.state
 
     @property
@@ -421,18 +404,19 @@ class Detector:
             f"prepare_acq({uuid}, {ctrl_params}, {acq_params}, {proc_params})"
         )
 
-        jsonschema.validate(
+        validate(
             instance=ctrl_params,
             schema=self.params_schema[self.ctrl.name()]["acq_params"],
         )
 
         for i, recv in enumerate(self.__recvs):
-            jsonschema.validate(
+            validate(
                 instance=acq_params[i],
                 schema=self.params_schema[recv.name()]["acq_params"],
             )
 
-        # TODO: Validate proc_params against the schema
+            pipeline_class = pipelines.get_class(proc_params[i]["class_name"])
+            validate(instance=proc_params[i], schema=pipeline_class.params_schema)
 
         # Update parameters on the device servers
         # TODO: Rename acq_params to ctrl_params

{lima2_client-2.0.2 → lima2_client-2.1.0}/src/lima2/client/devencoded/dense_frame.py

@@ -10,11 +10,13 @@
 
 from dataclasses import dataclass
 import struct
-import numpy
 import typing
 from typing_extensions import Union
 import enum
 
+import numpy
+from typing_extensions import Union
+
 from .exception import DevEncodedFormatNotSupported
 
 DATA_HEADER_FORMAT = "<IHHIIHHHHHHHHIIIIIIQ"
@@ -61,15 +63,9 @@ class Frame:
     data: numpy.ndarray
     """Data of the frame"""
 
-    idx: typing.Optional[int]
+    idx: int
     """Index of the frame where 0 is the first frame"""
 
-    def __bool__(self):
-        return not self.isEmpty()
-
-    def isEmpty(self):
-        return self.data is None
-
 
 def decode(raw_data: EncodedFrame) -> Frame:
     """Decode data provided by Lima device image attribute

{lima2_client-2.0.2 → lima2_client-2.1.0}/src/lima2/client/devencoded/sparse_frame.py

@@ -31,7 +31,7 @@ class SparseFrame:
     index: np.ndarray
     intensity: np.ndarray
 
-    idx: int | None
+    idx: int
     """Index of the frame where 0 is the first frame"""
 
     shape: tuple[int, ...]

{lima2_client-2.0.2 → lima2_client-2.1.0}/src/lima2/client/pipeline.py

@@ -14,6 +14,7 @@ frame index, and provide aggregated progress counters during/after an acquisitio
 
 import json
 import logging
+import traceback
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
@@ -36,7 +37,6 @@ from lima2.client.topology import (
 )
 from lima2.client.utils import classproperty
 
-# Create a logger
 logger = logging.getLogger(__name__)
 
 
@@ -231,14 +231,29 @@ class Pipeline:
             # The most naive way to get the latest frame is to try all receivers.
             # For monitoring purposes (~1 Hz), this is OK, but would be very wasteful otherwise.
             num_rcvs = len(self._devs)
+
+            def fetch_and_decode_last(rcv_idx):
+                """Return the decoded last frame from receiver rcv_idx.
+
+                Returns None if the receiver hasn't yet processed any frames.
+                """
+                try:
+                    return frame_source.decoder(getters[rcv_idx](-1))
+                except tango.DevFailed:
+                    return None
+
             with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
                 frames = list(
                     pool.map(
-                        lambda rcv_idx: frame_source.decoder(getters[rcv_idx](-1)),
+                        fetch_and_decode_last,
                         range(num_rcvs),
                     )
                 )
-            indices = [frame.idx for frame in frames]
+            indices = [frame.idx for frame in frames if frame is not None]
+            if len(indices) == 0:
+                raise ValueError(
+                    "Cannot get the latest frame: no frames available from any receiver"
+                )
             latest_idx = indices.index(max(indices))
 
             logger.debug(
@@ -505,22 +520,58 @@ class Pipeline:
         """A list of `is_finished` for each devices."""
         return [dev.is_finished for dev in self._devs]
 
-    def register_on_finished(self, cbk):
+    def register_on_finished(self, on_finished: Callable[[], None]) -> None:
         """
-        Register a callback function to be notified on pipeline finish
+        Register a callback function to be notified when all pipelines are finished.
 
         Arg:
-            cbk: A callback `on_finished(evt: Tango.Event)` called for each receivers
-
-        Returns:
-            A dict mapping the pipeline instance name with the event id
+            on_finished: A callback called when all receivers are done
+              processing frames.
         """
 
-        return {
-            proc: proc.subscribe_event(
-                "is_finished", tango.EventType.DATA_READY_EVENT, cbk
+        self.finished_devs: set[tango.DeviceProxy] = set()
+        """Set of processing devices that have finished."""
+
+        event_ids: dict[tango.DeviceProxy, int] = {}
+
+        def device_finished_callback(evt: tango.DataReadyEventData):
+            nonlocal event_ids
+
+            if evt.err:
+                logger.error(
+                    f"Error on {evt.device.dev_name()} 'is_finished' event:\n"
+                    + "\n".join([f"- {err.desc}" for err in evt.errors])
+                    + "\n"
+                )
+                for r, e in event_ids.items():
+                    r.unsubscribe_event(e)
+                return
+
+            logger.debug(f"{evt.device.dev_name()} is_finished event received")
+
+            self.finished_devs.add(evt.device)
+            if self.finished_devs == {dev for dev in self._devs}:
+                # All done. Call on_finished and report any exceptions.
+                try:
+                    on_finished()
+                except Exception:
+                    logger.error(
+                        "Exception raised in pipeline on_finished callback:\n"
+                        f"{traceback.format_exc()}"
+                    )
+
+                for dev, event_id in event_ids.items():
+                    # Unsubscribe from the event to stop trying in case of error
+                    logger.debug(f"Unsubscribing from {event_id} on {dev.dev_name()}")
+                    dev.unsubscribe_event(event_id)
+
+        event_ids = {
+            dev: dev.subscribe_event(
+                "is_finished",
+                tango.EventType.DATA_READY_EVENT,
+                device_finished_callback,
             )
-            for proc in self._devs
+            for dev in self._devs
         }
 
     @property
@@ -539,9 +590,31 @@ class Pipeline:
             A dict mapping the pipeline instance name with the event id
         """
 
-        return {
+        event_ids: dict[tango.DeviceProxy, int] = {}
+
+        def on_error(evt: tango.DataReadyEventData):
+            nonlocal event_ids
+
+            if evt.err:
+                logger.error(
+                    f"Error on {evt.device.dev_name()} 'last_error' event:\n"
+                    + "\n".join([f"- {err.desc}" for err in evt.errors])
+                    + "\n"
+                )
+            else:
+                try:
+                    cbk(evt)
+                except Exception:
+                    logger.error(
+                        "Exception raised in pipeline on_error callback:\n"
+                        f"{traceback.format_exc()}"
+                    )
+
+            evt.device.unsubscribe_event(event_ids[evt.device])
+
+        event_ids = {
             proc: proc.subscribe_event(
-                "last_error", tango.EventType.DATA_READY_EVENT, cbk
+                "last_error", tango.EventType.DATA_READY_EVENT, on_error
             )
             for proc in self._devs
         }

{lima2_client-2.0.2 → lima2_client-2.1.0}/src/lima2/client/shell.py

@@ -33,7 +33,6 @@ def get_proc_params(pipeline: Literal["Legacy", "Smx", "Xpcs"]) -> dict:
 
     proc_class = pipelines.get_class(f"LimaProcessing{pipeline}")
     proc_params: dict = proc_class.params_default
-    proc_params["class_name"] = proc_class.tango_class
 
     return proc_params
 
@@ -64,6 +63,7 @@ def run_acquisition(
 
 def main():
     import copy
+    import logging
     import os
 
     import tango as tg
@@ -83,6 +83,8 @@ def main():
     if not os.getenv("TANGO_HOST"):
         raise ValueError("TANGO_HOST must be exported")
 
+    logging.basicConfig(level=logging.INFO)
+
     #############
     # Populate user namespace
 
@@ -103,7 +105,7 @@ def main():
     proc_params = get_proc_params("Legacy")
     proc_params["saving"]["file_exists_policy"] = "overwrite"
 
-    ctl_params = c.params_default[c.detector.ctrl.name()]["acq_params"]
+    ctl_params = c.params_default[c.control.name()]["acq_params"]
 
     rcv_params = copy.deepcopy(ctl_params)