lima2.client 3.0a1__tar.gz → 3.1a0__tar.gz

{lima2_client-3.0a1/src/lima2_client.egg-info → lima2_client-3.1a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lima2-client
-Version: 3.0a1
+Version: 3.1a0
 Summary: Python client library for Lima2 detector
 Author: BCU (ESRF)
 License: Copyright (c) 2020-2024 Beamline Control Unit, ESRF

{lima2_client-3.0a1 → lima2_client-3.1a0}/src/lima2/client/client.py

@@ -34,6 +34,8 @@ from lima2.client.progress_counter import ProgressCounter, SingleCounter
 from lima2.client.topology import TopologyKind
 from lima2.client.utils import docstring_from
 
+from lima2.master_file import generate as mfg
+
 logger = logging.getLogger(__name__)
 
 
@@ -77,6 +79,16 @@ class Client:
 
         self._tango_db = tg.Database()
 
+        # Show master file generation info messages
+        mfg.logger.setLevel(logging.INFO)
+
+        self.master_file_tasks: dict[UUID, dict[str, mfg.MasterFileTask]] = {}
+        """Master file generation tasks for each pipeline UUID.
+
+        Each entry corresponds to a pipeline, and holds a dict structured like
+        {saving channel name: list of tasks}.
+        """
+
     @classmethod
     def from_yaml(
         cls,
@@ -147,7 +159,11 @@ class Client:
     ####################################################################################
 
     def prepare_acq(
-        self, uuid: UUID, ctrl_params: dict, acq_params: dict, proc_params: dict
+        self,
+        uuid: UUID,
+        ctrl_params: dict,
+        acq_params: dict,
+        proc_params: dict,
     ) -> None:
         """Prepare acquisition with a given uuid and set of params."""
 
@@ -175,16 +191,40 @@ class Client:
 
         self.current_pipeline.register_on_error(on_error)
 
+        # Remove master file tasks of previous acquisitions if done or not started
+        for prev_uuid in list(self.master_file_tasks.keys()):
+            tasks = self.master_file_tasks[prev_uuid].values()
+            if all([not task.started() or task.ready() for task in tasks]):
+                logger.debug(
+                    "All master file tasks done (or never started) for "
+                    f"{prev_uuid} -> cleaning up"
+                )
+                self.master_file_tasks.pop(prev_uuid)
+            else:
+                logger.debug(f"Not all master file tasks done for {prev_uuid}...")
+
+        self.master_file_tasks[uuid] = mfg.enable_generation(
+            pipeline=self.current_pipeline,
+            num_frames=ctrl_params["acq"]["nb_frames"],
+            ctrl_params=ctl,
+            recv_params=acq[0],
+            proc_params=proc[0],
+        )
+
     @docstring_from(Detector.start_acq)
     def start_acq(self) -> None:
         pipeline = self.current_pipeline
         pipeline.register_on_finished(pipeline.unregister_events)
+
         try:
             self.detector.start_acq()
         except Exception:
             pipeline.unregister_events()
             raise
 
+        for task in self.master_file_tasks[self.current_pipeline.uuid].values():
+            task.start()
+
     @docstring_from(Detector.trigger)
     def trigger(self) -> None:
         self.detector.trigger()

{lima2_client-3.0a1 → lima2_client-3.1a0}/src/lima2/client/pipeline.py

@@ -379,13 +379,13 @@ class Pipeline:
 
         # Pop stats for new frames from each receiver and concatenate, mixing in cached data.
         # Data comes from server as 1D array with size (num_frames * num_rois)
-        new_data = np.concatenate(
-            [cache.flatten()]
-            + [
-                structured_array.decode(dev.popRoiStatistics(), dtype)
-                for dev in self._devs
-            ]
-        )
+        def pop_data(dev) -> np.ndarray:
+            return structured_array.decode(dev.popRoiStatistics(), dtype)
+
+        num_rcvs = len(self._devs)
+        with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
+            flat_cache = [cache.flatten()]
+            new_data = np.concatenate(flat_cache + list(pool.map(pop_data, self._devs)))
 
         # Guess number of frames from the data
         num_frames = np.unique(new_data["frame_idx"]).size
@@ -473,13 +473,13 @@ class Pipeline:
 
         # Pop profiles for new frames from each receiver and concatenate, mixing in cached data.
         # Data comes from server as 1D array with size (num_frames * sum(roi_lengths))
-        new_data = np.concatenate(
-            [cache.flatten()]
-            + [
-                structured_array.decode(dev.popRoiProfiles(), dtype)
-                for dev in self._devs
-            ]
-        )
+        def pop_data(dev) -> np.ndarray:
+            return structured_array.decode(dev.popRoiProfiles(), dtype)
+
+        num_rcvs = len(self._devs)
+        with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
+            flat_cache = [cache.flatten()]
+            new_data = np.concatenate(flat_cache + list(pool.map(pop_data, self._devs)))
 
         # Reshape into (num_frames, sum(roi_lengths))
         new_data = new_data.reshape((-1, sum(roi_lengths)))
@@ -654,9 +654,9 @@ class Pipeline:
                 "no associated label"
             )
 
-        def last_frames(rcv_idx):
+        def last_frames(rcv_idx: int) -> int:
             last_frames_str = self._devs[rcv_idx].last_frames
-            return json.loads(last_frames_str)[f"{source.label}_idx"]
+            return int(json.loads(last_frames_str)[f"{source.label}_idx"])
 
         num_rcvs = len(self._devs)
         with ThreadPoolExecutor(max_workers=num_rcvs) as pool:

{lima2_client-3.0a1 → lima2_client-3.1a0}/src/lima2/client/topology.py

@@ -9,6 +9,7 @@ import copy
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from enum import Enum
+from concurrent.futures import ThreadPoolExecutor
 
 import tango as tg
 from beartype import beartype
@@ -107,18 +108,22 @@ class UnevenTopology(Topology):
         If the frame is found on multiple devices (e.g. frame_idx=-1),
         return the one found last (not necessarily the most recent).
         """
-        frame: EncodedFrame | None = None
 
-        for i in range(self.num_receivers):
+        def get_frame(rcv_idx) -> EncodedFrame | None:
             try:
-                frame = getters[i](frame_idx)
+                frame: EncodedFrame = getters[rcv_idx](frame_idx)
+                return frame
             except tg.DevFailed:
-                pass
+                return None
 
-        if frame is None:
+        num_rcvs = self.num_receivers
+        with ThreadPoolExecutor(max_workers=num_rcvs) as pool:
+            frames = [f for f in pool.map(get_frame, range(num_rcvs)) if f is not None]
+
+        if len(frames) == 0:
             raise IndexError(f"Frame {frame_idx} not available from any receiver")
 
-        return frame
+        return frames[0]
 
 
 def distribute_acq(

{lima2_client-3.0a1 → lima2_client-3.1a0}/src/lima2/master_file/generate.py

@@ -13,7 +13,6 @@ processing pipelines finish.
 import json
 import logging
 import os
-import sys
 import time
 import traceback
 from copy import deepcopy
@@ -25,6 +24,7 @@ import h5py
 import numpy as np
 
 import gevent
+import gevent.event
 
 from lima2.client.pipeline import FrameType, Pipeline
 
@@ -39,13 +39,19 @@ class LUTSource(Enum):
 
 
 class MasterFileTask:
-    def __init__(self, task, end_event):
+    def __init__(self, task: gevent.Greenlet, end_event):
         self.task = task
         self.end_event = end_event
 
     def ready(self):
         return self.task.ready()
 
+    def start(self):
+        self.task.start()
+
+    def started(self) -> bool:
+        return self.task.started
+
     def wait(self, timeout):
         finished = gevent.wait([self.task], timeout=timeout)
         return len(finished) > 0
@@ -60,45 +66,6 @@ class MasterFileTask:
         self.task.join()
 
 
-active_tasks = {}
-
-
-def check_generation(
-    ctrl_params: dict, recv_params: dict, proc_params: dict
-) -> list[dict]:
-    """Check the parameters to be sent to prepare to ensure master file info"""
-
-    # group all active saving subsystems
-    active_saving_params: list[dict] = []
-
-    # first check receiver raw saving or SDK saving
-    for params in [recv_params, ctrl_params]:
-        if "saving" in params and params["saving"]["enabled"]:
-            active_saving_params.append(params["saving"])
-            break
-
-    # then check the saving streams of the current pipeline
-    for source, params in proc_params.items():
-        # Skip other subsystems
-        if not source.startswith("saving"):
-            continue
-        # Skip source if channel is disabled
-        if params["enabled"]:
-            active_saving_params.append(params)
-
-    # include detector frame_idx dataset in HDF5 files
-    for saving_params in active_saving_params:
-        if "include_frame_idx" in saving_params:
-            saving_params["include_frame_idx"] = True
-
-    # allow retreiving recv-det-frame-idx mapping
-    if "frame_idx_enabled" in proc_params:
-        logger.debug("Enabling frame_idx generation")
-        proc_params["frame_idx_enabled"] = True
-
-    return ctrl_params, recv_params, proc_params
-
-
 def enable_generation(
     pipeline: Pipeline,
     num_frames: int,
@@ -107,7 +74,7 @@ def enable_generation(
     proc_params: dict,
     rr_offset: int = 0,
     lut_source: LUTSource = LUTSource.AUTO,
-) -> None:
+) -> dict[str, MasterFileTask]:
     """Enable the generation of a master file for a given pipeline.
 
     Must be called between Client.prepare_acq() and Client.start_acq(), so that the
@@ -244,31 +211,29 @@ def enable_generation(
         logger.warning(
             "Master file generation not enabled: no dense frames saving channel enabled."
         )
-        return None
+        return {}
 
     def master_file_task(src, params) -> None:
-        try:
-            p = MasterFileGeneration.Params(**params)
-            gen = MasterFileGeneration(p)
-            gen.run()
-            logger.debug(f"On finished {src} master file generation")
-        except Exception as e:
-            logger.error("Error generating %s master file: %s", src, e)
-            traceback.print_exception(*sys.exc_info())
+        p = MasterFileGeneration.Params(**params)
+        gen = MasterFileGeneration(p)
+        gen.run()
+        logger.debug(f"On finished {src} master file generation")
 
-    tasks = {}
+    tasks: dict[str, MasterFileTask] = {}
 
     for src, params in generate_params.items():
-        task = gevent.spawn(master_file_task, src, params)
+        task = gevent.Greenlet(master_file_task, src, params)
         tasks[src] = MasterFileTask(task, end_event)
 
-        def cleanup(task):
-            if all([t.ready() for t in tasks.values()]):
-                active_tasks.pop(uuid)
-
-        task.link(cleanup)
+        def log_error(greenlet: gevent.Greenlet) -> None:
+            msg = (
+                f"Exception in master file task {str(uuid)} (channel '{src}'):\n"
+                + "".join(traceback.format_exception(greenlet.exception))
+            )
+            logger.error(msg)
+            print(msg)
 
-    active_tasks[uuid] = tasks
+        task.link_exception(callback=log_error)
 
     return tasks
 

{lima2_client-3.0a1 → lima2_client-3.1a0/src/lima2_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lima2-client
-Version: 3.0a1
+Version: 3.1a0
 Summary: Python client library for Lima2 detector
 Author: BCU (ESRF)
 License: Copyright (c) 2020-2024 Beamline Control Unit, ESRF

{lima2_client-3.0a1 → lima2_client-3.1a0}/src/lima2_client.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ src/lima2/client/pipelines/legacy.py
 src/lima2/client/pipelines/smx.py
 src/lima2/client/pipelines/xpcs.py
 src/lima2/master_file/generate.py
+src/lima2/master_file/py.typed
 src/lima2/server/__init__.py
 src/lima2/server/control.py
 src/lima2/server/receiver.py