lima2.client 3.0a1__tar.gz → 3.1.0__tar.gz

{lima2_client-3.0a1/src/lima2_client.egg-info → lima2_client-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lima2-client
-Version: 3.0a1
+Version: 3.1.0
 Summary: Python client library for Lima2 detector
 Author: BCU (ESRF)
 License: Copyright (c) 2020-2024 Beamline Control Unit, ESRF
@@ -24,7 +24,7 @@ Requires-Dist: gevent
 Requires-Dist: jsonschema
 Requires-Dist: jsonschema-default
 Requires-Dist: beartype
-Requires-Dist: numpy
+Requires-Dist: numpy<2.3
 Requires-Dist: pyyaml
 Requires-Dist: typing-extensions
 Requires-Dist: h5py

{lima2_client-3.0a1 → lima2_client-3.1.0}/pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "jsonschema",
     "jsonschema-default",
     "beartype",
-    "numpy",
+    "numpy<2.3",          # CI runner (CentOS 7) has glibc2.17, numpy>=2.3 needs glibc2.28+
     "pyyaml",
     "typing-extensions",
     "h5py",

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2/client/client.py

@@ -34,6 +34,8 @@ from lima2.client.progress_counter import ProgressCounter, SingleCounter
 from lima2.client.topology import TopologyKind
 from lima2.client.utils import docstring_from
 
+from lima2.master_file import generate as mfg
+
 logger = logging.getLogger(__name__)
 
 
@@ -77,6 +79,32 @@ class Client:
 
         self._tango_db = tg.Database()
 
+        # Show master file generation info messages
+        mfg.logger.setLevel(logging.INFO)
+
+        self.master_file_tasks: dict[UUID, dict[str, mfg.MasterFileTask]] = {}
+        """Master file generation tasks for each pipeline UUID.
+
+        Each entry corresponds to a pipeline, and holds a dict structured like
+        {saving channel name: list of tasks}.
+        """
+
+    def cleanup(self):
+        """Unregister all state loggers and tango event handlers.
+
+        This method serves only in specific cases where event handlers must for
+        some reason be unregistered. In a normal use case, a Client is
+        instantiated once per Lima2 instance, and no cleanup is necessary.
+
+        NOTE: After this call, the Client instance should no longer be used.
+        """
+        self._detector.unregister_events()
+        self._detector = None
+
+        for pipeline in self.__pipelines.values():
+            pipeline.unregister_events()
+        self.__pipelines.clear()
+
     @classmethod
     def from_yaml(
         cls,
@@ -147,10 +175,23 @@ class Client:
     ####################################################################################
 
     def prepare_acq(
-        self, uuid: UUID, ctrl_params: dict, acq_params: dict, proc_params: dict
+        self,
+        uuid: UUID,
+        ctrl_params: dict,
+        acq_params: dict,
+        proc_params: dict,
     ) -> None:
         """Prepare acquisition with a given uuid and set of params."""
 
+        # Check parameters required for master file generation
+        checked_params = mfg.check_generation(
+            ctrl_params=ctrl_params,
+            recv_params=acq_params,
+            proc_params=proc_params,
+            topology_kind=self.topology_kind,
+        )
+        ctrl_params, acq_params, proc_params = checked_params
+
         # Adjust params for distributed acquisition
         ctl, acq, proc = topology.distribute_acq(
             ctl_params=ctrl_params,
@@ -175,16 +216,41 @@ class Client:
 
         self.current_pipeline.register_on_error(on_error)
 
+        # Remove master file tasks of previous acquisitions if done or not started
+        for prev_uuid in list(self.master_file_tasks.keys()):
+            tasks = self.master_file_tasks[prev_uuid].values()
+            if all([not task.started() or task.ready() for task in tasks]):
+                logger.debug(
+                    "All master file tasks done (or never started) for "
+                    f"{prev_uuid} -> cleaning up"
+                )
+                self.master_file_tasks.pop(prev_uuid)
+            else:
+                logger.debug(f"Not all master file tasks done for {prev_uuid}...")
+
+        self.master_file_tasks[uuid] = mfg.enable_generation(
+            pipeline=self.current_pipeline,
+            num_frames=ctrl_params["acq"]["nb_frames"],
+            ctrl_params=ctl,
+            recv_params=acq[0],
+            proc_params=proc[0],
+            topology_kind=self.topology_kind,
+        )
+
     @docstring_from(Detector.start_acq)
     def start_acq(self) -> None:
         pipeline = self.current_pipeline
         pipeline.register_on_finished(pipeline.unregister_events)
+
         try:
             self.detector.start_acq()
         except Exception:
             pipeline.unregister_events()
             raise
 
+        for task in self.master_file_tasks[self.current_pipeline.uuid].values():
+            task.start()
+
     @docstring_from(Detector.trigger)
     def trigger(self) -> None:
         self.detector.trigger()

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2/client/detector.py

@@ -73,6 +73,7 @@ class Detector:
 
         self.__ctrl = ctrl_dev
         self.__recvs = list(recv_devs)
+        self._state_event_ids: dict[tango.DeviceProxy, int] = {}
 
         self.ping()
 
@@ -100,6 +101,17 @@ class Detector:
             d.set_green_mode(tango.GreenMode.Gevent)
             d.set_timeout_millis(timeout * 1000)
 
+    def unregister_events(self):
+        """Unregister all state loggers and tango event handlers.
+
+        This method serves only in specific cases where event handlers must for
+        some reason be unregistered. In a normal use case, a Client is
+        instantiated once per Lima2 instance, and no cleanup is necessary.
+        """
+        for device, event_id in self._state_event_ids.items():
+            device.unsubscribe_event(event_id)
+        self._state_event_ids.clear()
+
     @contextlib.contextmanager
     def _locked_fsm(self) -> Iterator[fsm.StateMachine]:
         self.__fsm_lock.acquire()
@@ -633,12 +645,14 @@ class Detector:
                     finally:
                         _logger.info(f"Current state: {fsm.state}")
 
-        for dev in self.devs:
-            dev.subscribe_event(
+        self._state_event_ids = {
+            dev: dev.subscribe_event(
                 "acq_state",
                 tango.EventType.CHANGE_EVENT,
                 on_acq_state_change,
             )
+            for dev in self.devs
+        }
 
     def _register_on_end_acq_handlers(self):
         """Register an "end of acquisition" callback on each receiver.

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2/client/pipeline.py

@@ -379,13 +379,13 @@ class Pipeline:
 
         # Pop stats for new frames from each receiver and concatenate, mixing in cached data.
         # Data comes from server as 1D array with size (num_frames * num_rois)
-        new_data = np.concatenate(
-            [cache.flatten()]
-            + [
-                structured_array.decode(dev.popRoiStatistics(), dtype)
-                for dev in self._devs
-            ]
-        )
+        def pop_data(dev) -> np.ndarray:
+            return structured_array.decode(dev.popRoiStatistics(), dtype)
+
+        num_rcvs = len(self._devs)
+        with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
+            flat_cache = [cache.flatten()]
+            new_data = np.concatenate(flat_cache + list(pool.map(pop_data, self._devs)))
 
         # Guess number of frames from the data
         num_frames = np.unique(new_data["frame_idx"]).size
@@ -473,13 +473,13 @@ class Pipeline:
 
         # Pop profiles for new frames from each receiver and concatenate, mixing in cached data.
         # Data comes from server as 1D array with size (num_frames * sum(roi_lengths))
-        new_data = np.concatenate(
-            [cache.flatten()]
-            + [
-                structured_array.decode(dev.popRoiProfiles(), dtype)
-                for dev in self._devs
-            ]
-        )
+        def pop_data(dev) -> np.ndarray:
+            return structured_array.decode(dev.popRoiProfiles(), dtype)
+
+        num_rcvs = len(self._devs)
+        with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
+            flat_cache = [cache.flatten()]
+            new_data = np.concatenate(flat_cache + list(pool.map(pop_data, self._devs)))
 
         # Reshape into (num_frames, sum(roi_lengths))
         new_data = new_data.reshape((-1, sum(roi_lengths)))
@@ -654,9 +654,9 @@ class Pipeline:
                 "no associated label"
             )
 
-        def last_frames(rcv_idx):
+        def last_frames(rcv_idx: int) -> int:
             last_frames_str = self._devs[rcv_idx].last_frames
-            return json.loads(last_frames_str)[f"{source.label}_idx"]
+            return int(json.loads(last_frames_str)[f"{source.label}_idx"])
 
         num_rcvs = len(self._devs)
         with ThreadPoolExecutor(max_workers=num_rcvs) as pool:

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2/client/state_machine.py

@@ -239,6 +239,9 @@ class StateMachine(fsm.StateMachine):
         if logger in self.__loggers:
             self.__loggers.remove(logger)
 
+    def clear_transition_loggers(self):
+        self.__loggers.clear()
+
     def on_state_change(self, state):
         _logger.debug(f"on_state_change {state}")
         futures = []

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2/client/topology.py

@@ -9,6 +9,7 @@ import copy
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from enum import Enum
+from concurrent.futures import ThreadPoolExecutor
 
 import tango as tg
 from beartype import beartype
@@ -107,18 +108,22 @@ class UnevenTopology(Topology):
         If the frame is found on multiple devices (e.g. frame_idx=-1),
         return the one found last (not necessarily the most recent).
         """
-        frame: EncodedFrame | None = None
 
-        for i in range(self.num_receivers):
+        def get_frame(rcv_idx) -> EncodedFrame | None:
             try:
-                frame = getters[i](frame_idx)
+                frame: EncodedFrame = getters[rcv_idx](frame_idx)
+                return frame
             except tg.DevFailed:
-                pass
+                return None
 
-        if frame is None:
+        num_rcvs = self.num_receivers
+        with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
+            frames = [f for f in pool.map(get_frame, range(num_rcvs)) if f is not None]
+
+        if len(frames) == 0:
             raise IndexError(f"Frame {frame_idx} not available from any receiver")
 
-        return frame
+        return frames[0]
 
 
 def distribute_acq(

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2/master_file/generate.py

@@ -13,7 +13,6 @@ processing pipelines finish.
 import json
 import logging
 import os
-import sys
 import time
 import traceback
 from copy import deepcopy
@@ -25,8 +24,10 @@ import h5py
 import numpy as np
 
 import gevent
+import gevent.event
 
 from lima2.client.pipeline import FrameType, Pipeline
+from lima2.client.topology import TopologyKind
 
 logger = logging.getLogger(__name__)
 
@@ -39,13 +40,19 @@ class LUTSource(Enum):
 
 
 class MasterFileTask:
-    def __init__(self, task, end_event):
+    def __init__(self, task: gevent.Greenlet, end_event):
         self.task = task
         self.end_event = end_event
 
     def ready(self):
         return self.task.ready()
 
+    def start(self):
+        self.task.start()
+
+    def started(self) -> bool:
+        return self.task.started
+
     def wait(self, timeout):
         finished = gevent.wait([self.task], timeout=timeout)
         return len(finished) > 0
@@ -60,11 +67,11 @@ class MasterFileTask:
         self.task.join()
 
 
-active_tasks = {}
-
-
 def check_generation(
-    ctrl_params: dict, recv_params: dict, proc_params: dict
+    ctrl_params: dict,
+    recv_params: dict,
+    proc_params: dict,
+    topology_kind: TopologyKind,
 ) -> list[dict]:
     """Check the parameters to be sent to prepare to ensure master file info"""
 
@@ -86,13 +93,15 @@ def check_generation(
         if params["enabled"]:
             active_saving_params.append(params)
 
+    need_frame_lut = topology_kind == TopologyKind.UNEVEN
+
     # include detector frame_idx dataset in HDF5 files
     for saving_params in active_saving_params:
-        if "include_frame_idx" in saving_params:
+        if need_frame_lut and "include_frame_idx" in saving_params:
             saving_params["include_frame_idx"] = True
 
     # allow retreiving recv-det-frame-idx mapping
-    if "frame_idx_enabled" in proc_params:
+    if need_frame_lut and "frame_idx_enabled" in proc_params:
         logger.debug("Enabling frame_idx generation")
         proc_params["frame_idx_enabled"] = True
 
@@ -105,9 +114,10 @@ def enable_generation(
     ctrl_params: dict,
     recv_params: dict,
     proc_params: dict,
+    topology_kind: TopologyKind,
     rr_offset: int = 0,
     lut_source: LUTSource = LUTSource.AUTO,
-) -> None:
+) -> dict[str, MasterFileTask]:
     """Enable the generation of a master file for a given pipeline.
 
     Must be called between Client.prepare_acq() and Client.start_acq(), so that the
@@ -177,6 +187,8 @@ def enable_generation(
         proc_params=proc_params,
     )
 
+    need_frame_lut = topology_kind == TopologyKind.UNEVEN
+
     for source, frame_info, saving_params, cnt_name in active_saving_params:
         base_path = saving_params["base_path"]
         filename_prefix = saving_params["filename_prefix"]
@@ -204,10 +216,12 @@ def enable_generation(
                 this_lut_source = LUTSource.ROUND_ROBIN
         if this_lut_source is None:
             raise ValueError(f"Cannot satisfy {lut_source=}: {file_lut=}, {recv_lut=}")
+        elif this_lut_source == LUTSource.ROUND_ROBIN and need_frame_lut:
+            raise ValueError(f"Cannot use {this_lut_source} with {topology_kind}")
 
         logger.info(
             f"Master file (channel={source}) will be generated "
-            f"at {master_file_path}..."
+            f"at {master_file_path} [{this_lut_source}]..."
         )
 
         if frame_info is not None:
@@ -244,31 +258,29 @@ def enable_generation(
         logger.warning(
             "Master file generation not enabled: no dense frames saving channel enabled."
         )
-        return None
+        return {}
 
     def master_file_task(src, params) -> None:
-        try:
-            p = MasterFileGeneration.Params(**params)
-            gen = MasterFileGeneration(p)
-            gen.run()
-            logger.debug(f"On finished {src} master file generation")
-        except Exception as e:
-            logger.error("Error generating %s master file: %s", src, e)
-            traceback.print_exception(*sys.exc_info())
+        p = MasterFileGeneration.Params(**params)
+        gen = MasterFileGeneration(p)
+        gen.run()
+        logger.debug(f"On finished {src} master file generation")
 
-    tasks = {}
+    tasks: dict[str, MasterFileTask] = {}
 
     for src, params in generate_params.items():
-        task = gevent.spawn(master_file_task, src, params)
+        task = gevent.Greenlet(master_file_task, src, params)
         tasks[src] = MasterFileTask(task, end_event)
 
-        def cleanup(task):
-            if all([t.ready() for t in tasks.values()]):
-                active_tasks.pop(uuid)
-
-        task.link(cleanup)
+        def log_error(greenlet: gevent.Greenlet) -> None:
+            msg = (
+                f"Exception in master file task {str(uuid)} (channel '{src}'):\n"
+                + "".join(traceback.format_exception(greenlet.exception))
+            )
+            logger.error(msg)
+            print(msg)
 
-    active_tasks[uuid] = tasks
+        task.link_exception(callback=log_error)
 
     return tasks
 

{lima2_client-3.0a1 → lima2_client-3.1.0/src/lima2_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lima2-client
-Version: 3.0a1
+Version: 3.1.0
 Summary: Python client library for Lima2 detector
 Author: BCU (ESRF)
 License: Copyright (c) 2020-2024 Beamline Control Unit, ESRF
@@ -24,7 +24,7 @@ Requires-Dist: gevent
 Requires-Dist: jsonschema
 Requires-Dist: jsonschema-default
 Requires-Dist: beartype
-Requires-Dist: numpy
+Requires-Dist: numpy<2.3
 Requires-Dist: pyyaml
 Requires-Dist: typing-extensions
 Requires-Dist: h5py

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2_client.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ src/lima2/client/pipelines/legacy.py
 src/lima2/client/pipelines/smx.py
 src/lima2/client/pipelines/xpcs.py
 src/lima2/master_file/generate.py
+src/lima2/master_file/py.typed
 src/lima2/server/__init__.py
 src/lima2/server/control.py
 src/lima2/server/receiver.py

{lima2_client-3.0a1 → lima2_client-3.1.0}/src/lima2_client.egg-info/requires.txt

@@ -3,7 +3,7 @@ gevent
 jsonschema
 jsonschema-default
 beartype
-numpy
+numpy<2.3
 pyyaml
 typing-extensions
 h5py