lima2.client 2.1.0__tar.gz → 2.1.1__tar.gz

{lima2_client-2.1.0/src/lima2.client.egg-info → lima2_client-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lima2.client
-Version: 2.1.0
+Version: 2.1.1
 Summary: Python client library for Lima2 detector
 Home-page: https://gitlab.esrf.fr/limagroup/lima2-client
 Author: BCU (ESRF)

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/client/pipeline.py

@@ -18,15 +18,17 @@ import traceback
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
+from enum import Enum, auto
 from uuid import UUID
 
 import numpy as np
 import tango
 from jsonschema_default import create_from
+from typing_extensions import Optional
 
 from lima2.client import progress_counter, utils
 from lima2.client.convert import frame_info_to_shape_dtype
-from lima2.client.devencoded import structured_array
+from lima2.client.devencoded import dense_frame, sparse_frame, structured_array
 from lima2.client.progress_counter import ProgressCounter, SingleCounter
 from lima2.client.topology import (
     RoundRobinTopology,
@@ -40,29 +42,44 @@ from lima2.client.utils import classproperty
 logger = logging.getLogger(__name__)
 
 
+class FrameType(Enum):
+    DENSE = auto()
+    SPARSE = auto()
+
+
+decoder_by_type: dict[FrameType, Callable] = {
+    FrameType.DENSE: dense_frame.decode,
+    FrameType.SPARSE: sparse_frame.decode,
+}
+"""Mapping from frame type to associated decode function"""
+
+
 @dataclass
 class FrameSource:
-    """Specifies the name of a getter and a decode function for a given frame source.
+    """Specifies the name of a getter and a frame type for a given frame source.
 
     From these two attributes, it is possible to fetch a frame from a processing tango
     device and then decode it.
     """
 
     getter_name: str
-    decoder: Callable
-
+    """Name of the getter method to call on the tango device to fetch the data"""
 
-class Pipeline:
-    """A base class for all processing pipelines."""
+    frame_type: FrameType
+    """Frame type"""
 
-    SAVING_PARAMS_NAMES: list[str]
-    """Names of members of type "saving_params" in the corresponding lima2 "proc_params" struct.
+    saving_channel: Optional[str]
+    """Name of the associated saving_params object in the proc_params struct.
 
-    Definition in child classes is enforced by __init_subclass__().
+    Can be None for sources without persistency.
     """
 
+
+class Pipeline:
+    """A base class for all processing pipelines."""
+
     FRAME_SOURCES: dict[str, FrameSource]
-    """Map of available frame source names to a corresponding getter name and decode function.
+    """Map of available frame source names to a corresponding FrameSource descriptor.
 
     Definition in child classes is enforced by __init_subclass__().
     """
@@ -128,11 +145,6 @@ class Pipeline:
     @classmethod
     def __init_subclass__(cls) -> None:
         """Initialize a pipeline subclass."""
-        if not hasattr(cls, "SAVING_PARAMS_NAMES"):
-            raise ValueError(
-                f"Pipeline subclass {cls} must define a SAVING_PARAMS_NAMES class member"
-            )
-
         if not hasattr(cls, "FRAME_SOURCES"):
             raise ValueError(
                 f"Pipeline subclass {cls} must define a FRAME_SOURCES class member"
@@ -225,6 +237,13 @@ class Pipeline:
                 f"Available ones are: {list(self.FRAME_SOURCES.keys())}"
             )
 
+        try:
+            decode = decoder_by_type[frame_source.frame_type]
+        except KeyError as e:
+            raise NotImplementedError(
+                f"Frame type {frame_source.frame_type} decoder missing"
+            ) from e
+
         getters = [getattr(dev, frame_source.getter_name) for dev in self._devs]
 
         if frame_idx == -1:
@@ -238,7 +257,7 @@ class Pipeline:
                 Returns None if the receiver hasn't yet processed any frames.
                 """
                 try:
-                    return frame_source.decoder(getters[rcv_idx](-1))
+                    return decode(getters[rcv_idx](-1))
                 except tango.DevFailed:
                     return None
 
@@ -264,7 +283,7 @@ class Pipeline:
             # Get a frame by absolute index -> use topology-specific lookup()
             frame = self.topology.lookup(frame_idx=frame_idx, getters=getters)
 
-            return frame_source.decoder(frame)
+            return decode(frame)
 
     def cache_byproduct(self, key: str, data: np.ndarray) -> None:
         """Store `data` into `self.byproduct_cache[key]`.

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/client/pipelines/legacy.py

@@ -13,8 +13,7 @@ from uuid import UUID
 import tango
 from beartype import beartype
 
-from lima2.client.devencoded import dense_frame
-from lima2.client.pipeline import FrameSource, Pipeline
+from lima2.client.pipeline import FrameSource, FrameType, Pipeline
 from lima2.client.topology import TopologyKind
 
 # Create a logger
@@ -25,14 +24,17 @@ _logger = logging.getLogger(__name__)
 class Legacy(Pipeline):
     tango_class = "LimaProcessingLegacy"
 
-    SAVING_PARAMS_NAMES = ["saving"]
-    """Members of type "saving_params" in the corresponding lima2 "proc_params" struct."""
-
     FRAME_SOURCES = {
         "input_frame": FrameSource(
-            getter_name="getInputFrame", decoder=dense_frame.decode
+            getter_name="getInputFrame",
+            frame_type=FrameType.DENSE,
+            saving_channel=None,
+        ),
+        "frame": FrameSource(
+            getter_name="getFrame",
+            frame_type=FrameType.DENSE,
+            saving_channel="saving",
         ),
-        "frame": FrameSource(getter_name="getFrame", decoder=dense_frame.decode),
     }
     """Available frame sources."""
 

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/client/pipelines/smx.py

@@ -16,10 +16,9 @@ import tango
 from lima2.client import utils
 from lima2.client.devencoded import (
     dense_frame,
-    sparse_frame,
     structured_array,
 )
-from lima2.client.pipeline import FrameSource, Pipeline
+from lima2.client.pipeline import FrameSource, FrameType, Pipeline
 from lima2.client.topology import TopologyKind
 
 # Create a logger
@@ -29,23 +28,27 @@ logger = logging.getLogger(__name__)
 class Smx(Pipeline):
     tango_class = "LimaProcessingSmx"
 
-    # Members of type "saving_params" in the corresponding lima2 "proc_params" struct.
-    SAVING_PARAMS_NAMES = [
-        "saving_dense",
-        "saving_sparse",
-        "saving_accumulation_corrected",
-        "saving_accumulation_peak",
-    ]
-
     FRAME_SOURCES = {
-        "frame": FrameSource(getter_name="getFrame", decoder=dense_frame.decode),
+        "frame": FrameSource(
+            getter_name="getFrame",
+            frame_type=FrameType.DENSE,
+            saving_channel="saving_dense",
+        ),
         "sparse_frame": FrameSource(
-            getter_name="getSparseFrame", decoder=sparse_frame.decode
+            getter_name="getSparseFrame",
+            frame_type=FrameType.SPARSE,
+            saving_channel="saving_sparse",
         ),
         "acc_corrected": FrameSource(
-            getter_name="getAccCorrected", decoder=dense_frame.decode
+            getter_name="getAccCorrected",
+            frame_type=FrameType.DENSE,
+            saving_channel="saving_accumulation_corrected",
+        ),
+        "acc_peaks": FrameSource(
+            getter_name="getAccPeaks",
+            frame_type=FrameType.DENSE,
+            saving_channel="saving_accumulation_peak",
         ),
-        "acc_peaks": FrameSource(getter_name="getAccPeaks", decoder=dense_frame.decode),
     }
     """Available frame sources."""
 

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/client/pipelines/xpcs.py

@@ -14,8 +14,8 @@ import numpy as np
 import tango
 
 from lima2.client import progress_counter, utils
-from lima2.client.devencoded import dense_frame, sparse_frame, structured_array
-from lima2.client.pipeline import FrameSource, Pipeline
+from lima2.client.devencoded import structured_array
+from lima2.client.pipeline import FrameSource, FrameType, Pipeline
 from lima2.client.topology import TopologyKind
 
 # Create a logger
@@ -25,19 +25,21 @@ logger = logging.getLogger(__name__)
 class Xpcs(Pipeline):
     tango_class = "LimaProcessingXpcs"
 
-    # Members of type "saving_params" in the corresponding lima2 "proc_params" struct.
-    SAVING_PARAMS_NAMES = [
-        "saving_dense",
-        "saving_sparse",
-    ]
-
     FRAME_SOURCES = {
         "input_frame": FrameSource(
-            getter_name="getInputFrame", decoder=dense_frame.decode
+            getter_name="getInputFrame",
+            frame_type=FrameType.DENSE,
+            saving_channel=None,
+        ),
+        "frame": FrameSource(
+            getter_name="getFrame",
+            frame_type=FrameType.DENSE,
+            saving_channel="saving_dense",
         ),
-        "frame": FrameSource(getter_name="getFrame", decoder=dense_frame.decode),
         "sparse_frame": FrameSource(
-            getter_name="getSparseFrame", decoder=sparse_frame.decode
+            getter_name="getSparseFrame",
+            frame_type=FrameType.SPARSE,
+            saving_channel="saving_sparse",
         ),
     }
     """Available frame sources."""

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/client/state_machine.py

@@ -115,18 +115,27 @@ class StateMachine(fsm.StateMachine):
         for device in [self.ctrl] + self.recvs:
             device.ping()
         states = [d.acq_state for d in [self.ctrl] + self.recvs]
-        if not all(s == states[0] for s in states):
+        ctl_state = states[0]
+
+        ServerState = type(ctl_state)
+
+        if all(s in [ServerState.idle, ServerState.prepared] for s in states):
+            # Tolerate a mix of prepared and idle devices. In this case,
+            # the global state shall be IDLE (forcing user to prepare again).
+            self.state = State.IDLE
+        elif not all(s == ctl_state for s in states):
+            # Otherwise, if states are divergent, sync cannot be done
             self._state = State.UNKNOWN
             raise RuntimeError(f"Cannot sync: inconsistent server states ({states})")
-        server_state = states[0]
-        ServerState = type(server_state)
-        map = {
-            ServerState.idle: State.IDLE,
-            ServerState.prepared: State.PREPARED,
-            ServerState.running: State.RUNNING,
-            ServerState.stopped: State.IDLE,
-        }
-        self.state = map[server_state]
+        else:
+            map = {
+                ServerState.idle: State.IDLE,
+                ServerState.prepared: State.PREPARED,
+                ServerState.running: State.RUNNING,
+                ServerState.stopped: State.IDLE,
+            }
+            self.state = map[ctl_state]
+
         return self.state
 
     @property

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/client/topology.py

@@ -152,8 +152,9 @@ def distribute_acq(
     for i in range(num_receivers):
         # Assign unique filename rank per receiver
         processing_class = pipelines.get_class(proc_params["class_name"])
-        for name in processing_class.SAVING_PARAMS_NAMES:
-            proc[i][name]["filename_rank"] = i
+        for source in processing_class.FRAME_SOURCES.values():
+            if source.saving_channel is not None:
+                proc[i][source.saving_channel]["filename_rank"] = i
 
         # Idem for detector RAW saving params
         if "saving" in acq[i]:

{lima2_client-2.1.0 → lima2_client-2.1.1}/src/lima2/master_file/generate.py

@@ -19,7 +19,7 @@ import os
 import h5py
 import numpy as np
 
-from lima2.client.pipeline import Pipeline
+from lima2.client.pipeline import FrameType, Pipeline
 
 logger = logging.getLogger(__name__)
 
@@ -42,42 +42,69 @@ def enable_generation(
         rr_offset (int): expected receiver of frame 0 in the round robin frame dispatching
     """
 
-    if proc_params["class_name"] != "LimaProcessingLegacy":
+    if rr_offset >= len(pipeline._devs):
         raise ValueError(
-            f"Unsupported master file generation for pipeline {proc_params['class_name']}"
+            f"Round robin offset cannot exceed the last receiver index ({len(pipeline._devs) - 1})."
         )
 
-    base_path = proc_params["saving"]["base_path"]
-    filename_prefix = proc_params["saving"]["filename_prefix"]
-    file_exists_policy = proc_params["saving"]["file_exists_policy"]
+    generate_params: list[dict] = []
+    """Params passed to generate_master_file for each enabled saving channel"""
 
-    master_file_path = f"{base_path}/{filename_prefix}_master.h5"
-    if os.path.exists(master_file_path) and file_exists_policy != "overwrite":
-        raise ValueError(
-            f"Master file exists at {master_file_path} but {file_exists_policy=}. "
-            "Aborting master file generation."
-        )
+    for name, source in pipeline.FRAME_SOURCES.items():
+        # Skip non-dense frame sources
+        if source.frame_type != FrameType.DENSE:
+            continue
+        # Skip non-persistent sources
+        if source.saving_channel is None:
+            continue
+        # Skip source if channel is disabled
+        if not proc_params[source.saving_channel]["enabled"]:
+            continue
 
-    if rr_offset >= len(pipeline._devs):
-        raise ValueError(
-            f"Round robin offset cannot exceed the last receiver index ({len(pipeline._devs) - 1})."
-        )
+        base_path = proc_params[source.saving_channel]["base_path"]
+        filename_prefix = proc_params[source.saving_channel]["filename_prefix"]
+        file_exists_policy = proc_params[source.saving_channel]["file_exists_policy"]
 
-    logger.info(f"Master file will be generated at {master_file_path}...")
+        master_file_path = f"{base_path}/{filename_prefix}_master.h5"
+        if os.path.exists(master_file_path) and file_exists_policy != "overwrite":
+            raise ValueError(
+                f"Master file (channel={source.saving_channel}) exists at {master_file_path} "
+                f"but {file_exists_policy=}. Cannot enable master file generation."
+            )
 
-    frame_info = pipeline.channels["frame"]
+        logger.info(
+            f"Master file (channel={source.saving_channel}) will be generated "
+            f"at {master_file_path}..."
+        )
 
-    def on_finished() -> None:
-        generate_master_file(
-            num_frames=num_frames,
-            frame_shape=frame_info["shape"],
-            pixel_dtype=frame_info["dtype"],
-            num_receivers=len(pipeline._devs),
-            saving_params=proc_params["saving"],
-            rr_offset=rr_offset,
+        frame_info = pipeline.channels[name]
+
+        # Calling register_on_finished here with a lambda doesn't work as expected
+        # because some parameters will be evaluated at the time of calling, and their value
+        # will have changed in subsequent loop iterations.
+        # One solution is to store the parameters in a dict and call register_on_finished
+        # outside of the loop.
+        generate_params.append(
+            dict(
+                num_frames=num_frames,
+                frame_shape=frame_info["shape"],
+                pixel_dtype=frame_info["dtype"],
+                num_receivers=len(pipeline._devs),
+                saving_params=proc_params[source.saving_channel],
+                rr_offset=rr_offset,
+            )
         )
 
-    pipeline.register_on_finished(on_finished=on_finished)
+    if len(generate_params) == 0:
+        logger.warning(
+            "Master file generation not enabled: no dense frames saving channel enabled."
+        )
+    else:
+        pipeline.register_on_finished(
+            on_finished=lambda: [
+                generate_master_file(**params) for params in generate_params
+            ]
+        )
 
 
 def generate_master_file(

{lima2_client-2.1.0 → lima2_client-2.1.1/src/lima2.client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: lima2.client
-Version: 2.1.0
+Version: 2.1.1
 Summary: Python client library for Lima2 detector
 Home-page: https://gitlab.esrf.fr/limagroup/lima2-client
 Author: BCU (ESRF)

{lima2_client-2.1.0 → lima2_client-2.1.1}/tests/test_pipeline.py

@@ -15,8 +15,8 @@ import numpy as np
 import pytest
 import tango
 
-from lima2.client import progress_counter
-from lima2.client.pipeline import FrameSource, Pipeline
+from lima2.client import pipeline, progress_counter
+from lima2.client.pipeline import FrameSource, FrameType, Pipeline
 from lima2.client.pipelines.smx import Smx
 from lima2.client.pipelines.xpcs import Xpcs
 from lima2.client.topology import TopologyKind
@@ -90,6 +90,20 @@ def test_constructor_uneven(processing_devices):
     )
 
 
+def test_bad_subclass():
+    with pytest.raises(ValueError):
+
+        class NoFrameSourcePipeline(Pipeline):
+            tango_class = "badcafe"
+            # Missing FRAME_SOURCES
+
+    with pytest.raises(ValueError):
+
+        class NoTangoClassPipeline(Pipeline):
+            FRAME_SOURCES = {}
+            # Missing tango_class
+
+
 def test_progress_counters(processing_devices):
     """Read progress_counters."""
     processing_devices[0].progress_counters = json.dumps(
@@ -387,7 +401,7 @@ def test_pop_roi_profiles(processing_devices):
     assert proc.byproduct_cache["roi_profiles"].size == 0  # No cache
 
 
-def test_get_frame(processing_devices):
+def test_get_frame(processing_devices, monkeypatch):
     proc = Pipeline(
         uuid=uuid1(),
         proc_devs=processing_devices,
@@ -403,10 +417,26 @@ def test_get_frame(processing_devices):
     proc.FRAME_SOURCES = {
         "cafedeca": FrameSource(
             getter_name="getFrame",
-            decoder=lambda raw: MockFrame(idx=int(raw[0]), data=raw[1:]),
-        )
+            frame_type=FrameType.DENSE,
+            saving_channel=None,
+        ),
+        "sparse": FrameSource(
+            getter_name="getSparseFrame",
+            frame_type=FrameType.SPARSE,
+            saving_channel=None,
+        ),
     }
 
+    def mock_decode(data: bytes):
+        return MockFrame(data[0], data[1:])
+
+    # Modify the decoder_by_type map to plug our mock decode function
+    # No decoder for SPARSE frame so we can test the case where the decoder is not implemented
+    monkeypatch.setattr(pipeline, "decoder_by_type", {FrameType.DENSE: mock_decode})
+
+    with pytest.raises(NotImplementedError):
+        proc.get_frame(0, source="sparse")
+
     rcv_0_frames = {
         0: b"\x00milk",
         -1: b"\x02milk",