lima2.client 4.2.0__tar.gz → 4.3.0__tar.gz

{lima2_client-4.2.0 → lima2_client-4.3.0}/.gitlab-ci.yml

@@ -88,6 +88,7 @@ integration-test:
   script:
     - mamba create -n lima2_env -c conda-forge -c esrf-bcu --yes
       "esrf-bcu::lima2-core>=1.0.0,<2"
+      conda-forge::pip
       conda-forge::git # for setuptools_scm
     - mamba run -n lima2_env pip install .[dev]
     - export LIMA2_PLUGIN_FOLDER="$(conda info --base)/envs/lima2_env/lib/plugins"
@@ -175,9 +176,8 @@ deploy-registry:
 
 pages:
   stage: doc
-  before_script:
-    - pip install .[docs]
   script:
+    - pip install .[docs]
     - mkdocs build
     - mv site public
   tags:

{lima2_client-4.2.0/src/lima2_client.egg-info → lima2_client-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lima2-client
-Version: 4.2.0
+Version: 4.3.0
 Summary: The Python client library of the Lima2 system
 Author: BCU (ESRF)
 License-Expression: MIT
@@ -17,6 +17,7 @@ Requires-Dist: packaging
 Requires-Dist: pytango
 Requires-Dist: requests
 Provides-Extra: conductor
+Requires-Dist: hdf5plugin; extra == "conductor"
 Requires-Dist: jsonschema-default; extra == "conductor"
 Requires-Dist: jsonschema-rs; extra == "conductor"
 Requires-Dist: orjson; extra == "conductor"

{lima2_client-4.2.0 → lima2_client-4.3.0}/conda/recipe.yaml

@@ -32,6 +32,7 @@ requirements:
     - termcolor # extra: shell
     - ipython # extra: shell
     - typer # extra: shell, conductor
+    - hdf5plugin # extra: conductor
     - jsonschema-default # extra: conductor
     - jsonschema-rs # extra: conductor
     - orjson # extra: conductor

lima2_client-4.3.0/docs/reference/accumulation.md

@@ -0,0 +1,13 @@
+# Accumulation
+
+::: lima2.client.accumulation
+
+    options:
+      toc_label: lima2.client.accumulation
+      show_root_heading: true
+      show_if_no_docstring: true
+      summary:
+        functions: true
+      filters:
+      - "!^_"
+      members_order: source

{lima2_client-4.2.0 → lima2_client-4.3.0}/mkdocs.yml

@@ -63,6 +63,7 @@ nav:
           - reference/services.md
           - reference/session.md
           - reference/exceptions.md
+          - reference/accumulation.md
   - contact.md
   - release_notes.md
   - Design docs:

{lima2_client-4.2.0 → lima2_client-4.3.0}/pyproject.toml

@@ -30,6 +30,7 @@ lima2-shell = "lima2.client.shell:app [dev]"
 
 [project.optional-dependencies]
 conductor = [
+    "hdf5plugin",
     "jsonschema-default",
     "jsonschema-rs",
     "orjson",

{lima2_client-4.2.0 → lima2_client-4.3.0}/src/lima2/client/_utils.py

@@ -6,8 +6,10 @@
 """Lima2 client private utility functions."""
 
 import logging
+from typing import Any
 
 import h5py
+import numpy as np
 import tango as tg
 
 from lima2.common import types
@@ -130,3 +132,17 @@ def hdf5_get_frame(
             raise NotImplementedError(repr(frame_type))
 
     return frm
+
+
+def jsonify_numpy_arrays(d: dict[str, Any]) -> dict[str, Any]:
+    """
+    Return an identical dictionary with numpy arrays replaced by JSON-ready objects.
+    """
+    ret: dict[str, Any] = {}
+    for k, v in d.items():
+        if isinstance(v, np.ndarray):
+            v = dict(type="numpy.ndarray", value=v.tolist(), dtype=str(v.dtype))
+        elif isinstance(v, dict):
+            v = jsonify_numpy_arrays(v)
+        ret[k] = v
+    return ret

lima2_client-4.3.0/src/lima2/client/accumulation.py

@@ -0,0 +1,150 @@
+# This file is part of the Lima2 project
+#
+# Copyright (c) 2020-2024 Beamline Control Unit, ESRF
+# Distributed under the MIT licence. See LICENSE for more info.
+
+"""Lima2 accumulation helpers."""
+
+import math
+from typing import Literal, NamedTuple
+
+
+class AccumulationParams(NamedTuple):
+    expo_time: float
+    nb_frames: int
+    nb_frames_per_trigger: int
+    acc_nb_frames: int
+
+
+def params_livetime(
+    expo_time: float,
+    max_expo_time: float,
+    npoints: int,
+    max_nb_frames_per_trigger: int,
+    trigger_mode: Literal["internal", "software", "external", "gate"],
+) -> AccumulationParams:
+    """
+    Calculate an optimal set of acquisition params given an upper limit constraint on
+    the duration of a single exposure, adding in accumulation if necessary, and if the
+    detector allows us to acquire a long-enough train of frames in a single trigger
+    pulse.
+
+    Assumes we are working in terms of "livetime", meaning the total exposure does not
+    include deadtime between frames. In this mode, accumulating `N` frames corresponds
+    to an integrated exposure of `N * subframe_expo`.
+
+    Args:
+      expo_time: Desired exposure of one accumulated frame (seconds).
+      max_expo_time: Desired upper limit on the expo time per subframe (seconds).
+      npoints: Number of scan iterations.
+      max_nb_frames_per_trigger: Maximum number of frames that the detector is able to
+        acquire per trigger signal.
+      trigger_mode: Whether the detector is triggered by hardware (external/gate),
+      software, or internally.
+    """
+    if max_expo_time <= 0:
+        raise ValueError("max_expo_time must be positive")
+    if expo_time <= 0:
+        raise ValueError("livetime must be positive")
+    if npoints < 0:
+        raise ValueError("npoints must not be negative")
+
+    if expo_time > max_expo_time:
+        acc_nb_frames = int(math.ceil(expo_time / max_expo_time))
+        det_expo_time = expo_time / acc_nb_frames
+        det_nb_frames = npoints * acc_nb_frames
+        nb_frames_per_trigger = acc_nb_frames
+
+        if acc_nb_frames > max_nb_frames_per_trigger:
+            if trigger_mode == "software":
+                raise ValueError(
+                    f"Can't resolve acquisition parameters in software triggering with "
+                    f"accumulation: "
+                    f"would need to set acc_nb_frames={acc_nb_frames} but the "
+                    f"upper limit is {max_nb_frames_per_trigger}."
+                )
+            elif trigger_mode == "external":
+                nb_frames_per_trigger = 1
+
+        return AccumulationParams(
+            expo_time=det_expo_time,
+            nb_frames=det_nb_frames,
+            nb_frames_per_trigger=nb_frames_per_trigger,
+            acc_nb_frames=acc_nb_frames,
+        )
+    else:
+        return AccumulationParams(
+            expo_time=expo_time,
+            nb_frames=npoints,
+            nb_frames_per_trigger=1,
+            acc_nb_frames=1,
+        )
+
+
+def params_realtime(
+    expo_time: float,
+    latency_time: float,
+    max_expo_time: float,
+    npoints: int,
+    max_nb_frames_per_trigger: int,
+    trigger_mode: Literal["internal", "software", "external", "gate"],
+) -> AccumulationParams:
+    """
+    Calculate an optimal set of acquisition params given an upper limit constraint on
+    the duration of a single exposure, adding in accumulation if necessary, and if the
+    detector allows us to acquire a long-enough train of frames in a single trigger
+    pulse.
+
+    Assumes we are working in terms of "realtime", where the resulting train of frames
+    should not exceed the requested frame time. To accomplish this, we must take into
+    account the dead time between exposures (latency time).
+
+    Args:
+      expo_time: Desired exposure of one accumulated frame (seconds).
+      latency_time: Delay between two subframe exposures (seconds).
+      max_expo_time: Desired upper limit on the expo time per subframe (seconds).
+      npoints: Number of scan iterations.
+      max_nb_frames_per_trigger: Maximum number of frames that the detector is able to
+        acquire per trigger signal.
+      trigger_mode: Whether the detector is triggered by hardware (external/gate),
+      software, or internally.
+    """
+    if max_expo_time <= 0:
+        raise ValueError("max_expo_time must be positive")
+    if expo_time <= 0:
+        raise ValueError("livetime must be positive")
+    if npoints < 0:
+        raise ValueError("npoints must not be negative")
+
+    if expo_time > max_expo_time:
+        acc_nb_frames = int(
+            math.ceil((expo_time + latency_time) / (max_expo_time + latency_time))
+        )
+        det_expo_time = (expo_time + latency_time) / acc_nb_frames - latency_time
+        det_nb_frames = npoints * acc_nb_frames
+        nb_frames_per_trigger = acc_nb_frames
+
+        if acc_nb_frames > max_nb_frames_per_trigger:
+            if trigger_mode == "software":
+                raise ValueError(
+                    f"Can't resolve acquisition parameters in software triggering with "
+                    f"accumulation: "
+                    f"would need to set acc_nb_frames={acc_nb_frames} but the "
+                    f"upper limit is {max_nb_frames_per_trigger}."
+                )
+            elif trigger_mode == "external":
+                nb_frames_per_trigger = 1
+
+        return AccumulationParams(
+            expo_time=det_expo_time,
+            nb_frames=det_nb_frames,
+            nb_frames_per_trigger=nb_frames_per_trigger,
+            acc_nb_frames=acc_nb_frames,
+        )
+    else:
+        return AccumulationParams(
+            expo_time=expo_time,
+            nb_frames=npoints,
+            nb_frames_per_trigger=1,
+            acc_nb_frames=1,
+        )

{lima2_client-4.2.0 → lima2_client-4.3.0}/src/lima2/client/services.py

@@ -32,6 +32,43 @@ from lima2.common.types import FrameInfo, FrameType
 logger = logging.getLogger(__name__)
 
 
+########################################################################################
+# Acquisition results
+########################################################################################
+
+
+@dataclass
+class AcquisitionSuccess:
+    """The acquisition completed successfully, despite potential warnings."""
+
+    warnings: list[str]
+
+
+@dataclass
+class AcquisitionFailure:
+    """The acquisition failed because or acquisition or processing issues."""
+
+    acquisition_errors: list[str]
+    """Fatal errors during frame acquisition."""
+    processing_errors: list[str]
+    """Fatal errors during frame processing."""
+    warnings: list[str]
+    """Non-critical issues that occurred during the acquisition."""
+
+
+class Unknown:
+    """No acquisition is ongoing, or the current acquisition is still running."""
+
+
+AcquisitionResult = AcquisitionSuccess | AcquisitionFailure | Unknown
+"""Return type of `Acquisition.result()`. Outcome of the latest acquisition."""
+
+
+########################################################################################
+# Services
+########################################################################################
+
+
 class Acquisition:
     """Conductor /acquisition service requests."""
 
@@ -56,7 +93,7 @@ class Acquisition:
                 "control": control,
                 "receiver": receiver,
                 "processing": processing,
-                "masterfile_metadata": masterfile_metadata,
+                "masterfile_metadata": _utils.jsonify_numpy_arrays(masterfile_metadata),
             },
         ).json()
         return UUID(uuid_str)
@@ -104,6 +141,30 @@ class Acquisition:
         runstate = RunState(self.session.get("/acquisition/state").json()["value"])
         return bool(runstate & RunState.FAULT)
 
+    def result(self) -> AcquisitionResult:
+        """Get information about the latest acquisition's outcome.
+
+        Returns an instance of either AcquisitionSuccess, AcquisitionFailure, or
+        Unknown.
+        """
+        result_json = self.session.get("/acquisition/result").json()
+        status = result_json["type"]
+
+        if status == "unknown":
+            return Unknown()
+        elif status == "success":
+            return AcquisitionSuccess(warnings=result_json["warnings"])
+        elif status == "failure":
+            acq_errors = result_json["errors"].get("acquisition", [])
+            proc_errors = result_json["errors"].get("processing", [])
+            return AcquisitionFailure(
+                acquisition_errors=acq_errors,
+                processing_errors=proc_errors,
+                warnings=result_json["warnings"],
+            )
+        else:
+            raise NotImplementedError(f"Unknown result format: {result_json}")
+
     def default_params(self) -> tuple[dict[str, Any], dict[str, Any]]:
         """Get default acquisition parameters.
 

{lima2_client-4.2.0 → lima2_client-4.3.0}/src/lima2/common/pipelines.py

@@ -84,7 +84,7 @@ smx = Pipeline(
         ),
     },
     reduced_data_sources={
-        "peak_counter": ScalarDataSource(
+        "nb_peaks": ScalarDataSource(
             getter_name="popPeakCounters",
             src_dtype=PEAK_COUNTER_DTYPE,
             channel_keys=["nb_peaks"],

{lima2_client-4.2.0 → lima2_client-4.3.0}/src/lima2/conductor/acquisition.py

@@ -15,6 +15,7 @@ import traceback
 import weakref
 from dataclasses import dataclass
 from enum import Enum, auto
+from pprint import pformat
 from typing import Awaitable, Callable
 
 from lima2.common.exceptions import Lima2BackendError
@@ -24,6 +25,7 @@ from lima2.conductor.processing.master_file import (
     FrameSavingChannel,
     MasterFileMetadata,
 )
+from lima2.conductor.processing.pipeline import Pipeline
 from lima2.conductor.processing.reduced_data import RoiParams
 from lima2.conductor.tango.container import TangoProcessingGroup
 from lima2.conductor.tango.processing import ProcessingErrorEvent, TangoProcessing
@@ -72,6 +74,9 @@ OnEndCallback = Callable[[SuccessFlag], Awaitable[None]]
 OnFailureCallback = Callable[[], Awaitable[None]]
 """Signature of the on_failure() callback."""
 
+OnProcessingFinishedCallback = Callable[[], Awaitable[None]]
+"""Signature of the on_processing_finished() callback."""
+
 
 class AcquisitionState:
     """Represents the state of a running acquisition.
@@ -94,13 +99,17 @@ class AcquisitionState:
         num_receivers: int,
         on_end: OnEndCallback,
         on_failure: OnFailureCallback,
+        on_processing_finished: OnProcessingFinishedCallback,
     ) -> None:
         self.num_receivers = num_receivers
 
-        self.on_end = on_end
+        # NOTE: use WeakMethods to avoid cyclic Acquisition <-> AcquisitionState refs
+        self.on_end_ref = weakref.WeakMethod(on_end)
         """Called when all devices are done (successfully or not)."""
-        self.on_failure = on_failure
+        self.on_failure_ref = weakref.WeakMethod(on_failure)
         """Called when any device fails, but only once."""
+        self.on_processing_finished_ref = weakref.WeakMethod(on_processing_finished)
+        """Called when all processing devices are finished (successfully or not)."""
 
         # Substate: we need to keep track of how many receivers and processing
         # devices are done, whether some failed, whether failure has already
@@ -247,6 +256,10 @@ class AcquisitionState:
             case AcquisitionEvent.PROC_FINISHED:
                 self.proc_finished = True
 
+                on_processing_finished = self.on_processing_finished_ref()
+                if on_processing_finished:
+                    await on_processing_finished()
+
                 if self.acq_finished:
                     events.append(AcquisitionEvent.FINISHED)
 
@@ -262,9 +275,13 @@ class AcquisitionState:
                     logger.info("🔥 Pipeline finished with errors: see above 🔥")
 
                 try:
-                    await self.on_end(
-                        SuccessFlag(success=not (self.rcv_failed or self.proc_failed))
-                    )
+                    on_end = self.on_end_ref()
+                    if on_end:
+                        await on_end(
+                            SuccessFlag(
+                                success=not (self.rcv_failed or self.proc_failed)
+                            )
+                        )
                 except BaseException:
                     logger.warning(
                         f"Exception in on_end() callback: {traceback.format_exc()}"
@@ -276,7 +293,9 @@ class AcquisitionState:
                 if not self.failed_once:
                     self.failed_once = True
                     try:
-                        await self.on_failure()
+                        on_failure = self.on_failure_ref()
+                        if on_failure:
+                            await on_failure()
                     except BaseException:
                         logger.warning(
                             f"Exception in on_failure() callback: "
@@ -336,6 +355,7 @@ class Acquisition:
         on_end: OnEndCallback,
         on_failure: OnFailureCallback,
         num_frames: int,
+        accumulation_depth: int,
         frame_infos: dict[str, FrameInfo],
         roi_params: RoiParams,
         saving_channels: dict[str, FrameSavingChannel],
@@ -346,6 +366,7 @@ class Acquisition:
             num_receivers=len(processing_devices),
             on_end=on_end,
             on_failure=on_failure,
+            on_processing_finished=self.on_processing_finished,
         )
         self.proc_group = processing_devices
         """Processing device group."""
@@ -358,6 +379,7 @@ class Acquisition:
         self.pipeline = pipeline_class(
             devices=processing_devices,
             num_frames=num_frames,
+            accumulation_depth=accumulation_depth,
             frame_infos=frame_infos,
             roi_params=roi_params,
             saving_channels=saving_channels,
@@ -366,14 +388,30 @@ class Acquisition:
         """Processing pipeline."""
 
         self.processing_errors: list[str] = []
-        """Holds processing error messages that occurred during the run, if any."""
+        """Holds processing error messages that occurred during the run, if any.
+
+        This list contains errors from the REMOTE processing devices.
+        """
 
-        self.done_devices: set[TangoProcessing] = set()
-        """Set of processing devices that have finished (successfully or not)."""
+        self.pipeline_result: Pipeline.PipelineResult | None = None
+        """Holds the return value of pipeline.close(), once processing is finished.
+
+        This result contains information about the LOCAL pipeline-related tasks, not the
+        online processing done by Lima2 backend devices.
+        """
+
+    async def on_processing_finished(self) -> None:
+        """Registered as the AcquisitionState's on_processing_finished() callback."""
+        logger.info("Closing pipeline")
+        self.pipeline_result = await self.pipeline.close()
+        logger.info(f"Pipeline result: {pformat(self.pipeline_result)}")
 
     def __del__(self) -> None:
         logger.debug(f"Acquisition instance {self.uuid} destroyed")
 
+    def started(self) -> bool:
+        return self.state.started
+
     def running(self) -> bool:
         return self.state.running()
 
@@ -391,6 +429,7 @@ class Acquisition:
         on_end: OnEndCallback,
         on_failure: OnFailureCallback,
         num_frames: int,
+        accumulation_depth: int,
         frame_infos: dict[str, FrameInfo],
         roi_params: RoiParams,
         saving_channels: dict[str, FrameSavingChannel],
@@ -405,6 +444,7 @@ class Acquisition:
             on_end=on_end,
             on_failure=on_failure,
             num_frames=num_frames,
+            accumulation_depth=accumulation_depth,
             frame_infos=frame_infos,
             roi_params=roi_params,
             saving_channels=saving_channels,
@@ -450,11 +490,6 @@ class Acquisition:
 
             await self_ref.notify(event=AcquisitionEvent.PROC_DEVICE_DONE)
 
-            self_ref.done_devices.add(device)
-            if self_ref.done_devices == set(self_ref.proc_group):
-                logger.info("Closing pipeline")
-                await self_ref.pipeline.close()
-
         async def on_device_error(evt: ProcessingErrorEvent) -> None:
             """Reports a processing error to the registered callback."""
             self_ref = wself()

{lima2_client-4.2.0 → lima2_client-4.3.0}/src/lima2/conductor/acquisition_system.py

@@ -385,6 +385,11 @@ class AcquisitionSystem:
             frame_types=frame_types,
         )
 
+        if "accumulation" in proc_params and proc_params["accumulation"]["enabled"]:
+            accumulation_depth = proc_params["accumulation"]["nb_frames"]
+        else:
+            accumulation_depth = 1
+
         logger.info(f"Frame infos:\n{pformat(frame_infos, indent=2)}")
         logger.info(f"Saving channels:\n{pformat(saving_channels, indent=2)}")
 
@@ -393,6 +398,7 @@ class AcquisitionSystem:
             on_end=self.on_acquisition_end,
             on_failure=self.on_acquisition_failure,
             num_frames=num_frames,
+            accumulation_depth=accumulation_depth,
             frame_infos=frame_infos,
             roi_params=roi_params,
             saving_channels=saving_channels,
@@ -620,6 +626,7 @@ class AcquisitionSystem:
             devices=proc_devices,
             frame_infos=frame_infos,
             num_frames=0,
+            accumulation_depth=1,
             roi_params=RoiParams(rois=[], profiles=[]),
             saving_channels={},
             masterfile_metadata=MasterFileMetadata(