lima2.client 4.1.8__tar.gz → 4.2.0__tar.gz

{lima2_client-4.1.8/src/lima2_client.egg-info → lima2_client-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lima2-client
-Version: 4.1.8
+Version: 4.2.0
 Summary: The Python client library of the Lima2 system
 Author: BCU (ESRF)
 License-Expression: MIT

{lima2_client-4.1.8 → lima2_client-4.2.0}/conda/recipe.yaml

@@ -50,6 +50,9 @@ tests:
         - lima2.client.services
         - lima2.common
       pip_check: true
+  - package_contents:
+      site_packages:
+        - lima2/conductor/webservice/monitor/*
   - script: lima2-shell --help
   - script: lima2-conductor --help
 

{lima2_client-4.1.8 → lima2_client-4.2.0}/docs/features.md

@@ -32,9 +32,18 @@ Both modes have a prefetched variant, where the frames are prefetched in memory
 
 Lima2 implements the following processing pipelines:
 
-- Legacy (aka Classic): a standard, CPU-based processing pipeline for generic image acquisition with optional [ROIs](./concepts.md#region-of-interest-roi).
-- [Smx (Serial Macromolecular Crystallography)](https://www.esrf.fr/id29): involves GPU-based processing to perform online azimuthal integration.
-- [XPCS (X-ray Photon Correlation Spectroscopy)](https://www.esrf.fr/UsersAndScience/Experiments/CBS/ID10): a pipeline dedicated to extracting temporal correlations from the images.
+- **Classic (aka Legacy)**: a general-purpose, CPU-based pipeline that can perform
+  standard image operations (cropping, rotation, background subtraction, etc.) as well
+  as [ROI statistics and profiles](./concepts.md#region-of-interest-roi).
+- **FAI (Fast Azimuthal Integration)**: specialized to process diffraction patterns, FAI
+  runs an online azimuthal integration step on GPU to efficiently find rings in
+  circularly-symmetric images.
+- [**Smx (Serial Macromolecular Crystallography)**](https://www.esrf.fr/id29): an
+  extended FAI pipeline, which additionally performs peak finding and sparsification after the
+  azimuthal integration step.
+- [**XPCS (X-ray Photon Correlation
+  Spectroscopy)**](https://www.esrf.fr/UsersAndScience/Experiments/CBS/ID10): a pipeline
+  dedicated to extracting temporal correlations from the images.
 
 Naturally, depending on the chosen processing type, the kinds of frames and
 [reduced data](./concepts.md#reduced-data) available to consumers changes.

lima2_client-4.2.0/docs/monitor.md

@@ -0,0 +1,12 @@
+# Status monitor
+
+The conductor serves a status monitoring webpage at its root endpoint ("/"). This page
+can be accessed in any browser, using the conductor's hostname and port.
+
+As long as the conductor is running, the page will periodically poll to update the
+displayed information.
+
+![Lima2 status webpage screenshot](assets/lima2-conductor-monitor-webpage.png)
+
+This page is mainly aimed at Lima2 administrators, and can be used to get an overview of
+the system's state (detector, receivers, ...).

lima2_client-4.2.0/src/lima2/client/_utils.py

@@ -0,0 +1,132 @@
+# This file is part of the Lima2 project
+#
+# Copyright (c) 2020-2024 Beamline Control Unit, ESRF
+# Distributed under the MIT licence. See LICENSE for more info.
+
+"""Lima2 client private utility functions."""
+
+import logging
+
+import h5py
+import tango as tg
+
+from lima2.common import types
+from lima2.common.devencoded.common import Frame
+from lima2.common.devencoded.data_array import Vector
+from lima2.common.devencoded.smx_sparse_frame import SmxSparseFrame
+from lima2.common.devencoded.sparse_frame import SparseFrame
+from lima2.common.types import FrameType
+
+logger = logging.getLogger(__name__)
+
+
+def tango_get_frame(
+    url: str,
+    command_name: str,
+    lookup_idx: int,
+) -> Frame | Vector | SparseFrame | SmxSparseFrame:
+    """Fetch a frame from a Lima2 receiver over tango, and decode it."""
+
+    device = tg.DeviceProxy(url)
+
+    try:
+        raw_data: tuple[str, bytes] = device.command_inout(
+            name=command_name, cmd_param=lookup_idx
+        )
+    except tg.DevFailed as e:
+        logger.error(
+            f"Call to {command_name}({lookup_idx}) on {device.dev_name()} failed"
+        )
+        raise RuntimeError(
+            f"Unable to get frame {lookup_idx} from {device.dev_name()}:\n{e}"
+        ) from e
+
+    try:
+        type_id, payload = raw_data
+    except ValueError:
+        raise RuntimeError(
+            f"Unexpected data received from Lima2 device: {raw_data!r}"
+        ) from None
+
+    if type_id not in types.decoder_by_id:
+        raise NotImplementedError(f"No decoder for '{type_id}'")
+
+    decoder = types.decoder_by_id[type_id]
+
+    return decoder(payload)
+
+
+def hdf5_get_frame(
+    url: str,
+    frame_idx: int,
+    lookup_idx: int,
+    frame_type: FrameType,
+    shape: tuple[int, ...],
+) -> Frame | Vector | SparseFrame | SmxSparseFrame:
+    """Fetch a frame from an HDF5 file."""
+    try:
+        filepath, datapath = url.removeprefix("file://").split(":")
+    except ValueError as e:
+        raise RuntimeError(
+            f"Invalid hdf5 url format: '{url}'. "
+            f"Expected 'file://file/path.h5:data/path"
+        ) from e
+
+    frm: Frame | Vector | SparseFrame | SmxSparseFrame
+
+    with h5py.File(filepath, "r") as f:
+        if frame_type == FrameType.DENSE:
+            try:
+                frm_data = f[f"{datapath}/data"][lookup_idx]
+            except IndexError as e:
+                raise RuntimeError(
+                    f"Can't access element {lookup_idx} at {url}."
+                ) from e
+            frm = Frame(data=frm_data, idx=frame_idx)
+
+        elif frame_type == FrameType.SPARSE:
+            try:
+                index = f[f"{datapath}/index"][lookup_idx]
+                intensity = f[f"{datapath}/intensity"][lookup_idx]
+            except IndexError as e:
+                raise RuntimeError(
+                    f"Can't access element {lookup_idx} at {url}."
+                ) from e
+            frm = SparseFrame(
+                index=index, intensity=intensity, idx=frame_idx, shape=shape
+            )
+
+        elif frame_type == FrameType.SMX_SPARSE:
+            try:
+                index = f[f"{datapath}/index"][lookup_idx]
+                intensity = f[f"{datapath}/intensity"][lookup_idx]
+                background_avg = f[f"{datapath}/background_avg"][lookup_idx]
+                background_std = f[f"{datapath}/background_std"][lookup_idx]
+            except IndexError as e:
+                raise RuntimeError(
+                    f"Can't access element {lookup_idx} at {url}."
+                ) from e
+
+            frm = SmxSparseFrame(
+                index=index,
+                intensity=intensity,
+                background_avg=background_avg,
+                background_std=background_std,
+                idx=frame_idx,
+                shape=shape,
+            )
+
+        elif frame_type == FrameType.VECTOR:
+            try:
+                vector_data = f[f"{datapath}/data"][lookup_idx]
+            except IndexError as e:
+                raise RuntimeError(
+                    f"Can't access element {lookup_idx} at {url}."
+                ) from e
+
+            frm = Vector(data=vector_data, idx=frame_idx)
+
+        else:
+            raise NotImplementedError(repr(frame_type))
+
+    return frm

{lima2_client-4.1.8 → lima2_client-4.2.0}/src/lima2/client/services.py

@@ -6,6 +6,7 @@
 """Conductor HTTP services wrapped into functions."""
 
 import logging
+import warnings
 from dataclasses import dataclass
 from enum import Enum, auto
 from importlib.metadata import version
@@ -14,19 +15,19 @@ from uuid import UUID
 
 import numpy as np
 import numpy.typing as npt
-import tango as tg
 from packaging.version import Version
 
+from lima2.client import _utils
 from lima2.client.exceptions import ConductorConnectionError, ConductorEndpointNotFound
 from lima2.client.session import ConductorSession
-from lima2.common import pipelines, types
-from lima2.common.devencoded.dense_frame import Frame
+from lima2.common.devencoded.common import Frame
+from lima2.common.devencoded.data_array import Vector
 from lima2.common.devencoded.smx_sparse_frame import SmxSparseFrame
 from lima2.common.devencoded.sparse_frame import SparseFrame
-from lima2.common.exceptions import Lima2LookupError, Lima2NotFound
+from lima2.common.exceptions import Lima2LookupError, Lima2NotFound, Lima2ValueError
 from lima2.common.progress_counter import ProgressCounter, SingleCounter
 from lima2.common.state import RunState, State
-from lima2.common.types import FrameInfo
+from lima2.common.types import FrameInfo, FrameType
 
 logger = logging.getLogger(__name__)
 
@@ -43,14 +44,19 @@ class Acquisition:
         control: dict[str, Any],
         receiver: dict[str, Any],
         processing: dict[str, Any],
+        masterfile_metadata: dict[str, Any] | None = None,
     ) -> UUID:
         """Prepare for a new acquisition given control, receiver and processing params."""
+        if masterfile_metadata is None:
+            masterfile_metadata = {}
+
         uuid_str = self.session.post(
             "/acquisition/prepare",
             json={
                 "control": control,
                 "receiver": receiver,
                 "processing": processing,
+                "masterfile_metadata": masterfile_metadata,
             },
         ).json()
         return UUID(uuid_str)
@@ -334,58 +340,90 @@ class Pipeline:
     def lookup(self, frame_idx: int, uuid: str | UUID = "current") -> tuple[int, str]:
         """Look up a frame by index.
 
+        Warning: Deprecated
+          This function will be removed in a future release.
+
         Returns a tuple (frame index, receiver url).
         """
+        warnings.warn(
+            "This function is deprecated and will be removed in a future release.",
+            stacklevel=2,
+        )
         json = self.session.get(f"/pipeline/{str(uuid)}/lookup/{frame_idx}").json()
 
         return (json["frame_idx"], json["receiver_url"])
 
     def get_frame(
         self, frame_idx: int, source: str, uuid: str | UUID = "current"
-    ) -> Frame | SparseFrame | SmxSparseFrame:
-        """Fetch a frame directly from a receiver device.
+    ) -> Frame | SparseFrame | SmxSparseFrame | Vector:
+        """Fetch a frame by absolute index.
 
-        Raises:
-          ValueError: Invalid frame source.
-          RuntimeError: The frame isn't available.
-        """
-        try:
-            fidx, rcv_url = self.lookup(frame_idx=frame_idx, uuid=uuid)
-        except Lima2LookupError as e:
-            raise RuntimeError(f"Can't find {source} {frame_idx}: {e}") from None
+        If frame_idx == -1, returns the most recent frame acquired.
 
-        class_name = self.pipeline(uuid=uuid)["type"]
-        try:
-            pipeline_info = pipelines.by_name[class_name]
-        except KeyError as e:
-            raise NotImplementedError(class_name) from e
+        Requires a source name, which depends on the type of processing pipeline.
+        Valid source names can be queried with
+        [`frame_channels()`][lima2.client.services.Pipeline.frame_channels].
 
-        try:
-            frame_source = pipeline_info.frame_sources[source]
-        except KeyError as e:
-            raise ValueError(
-                f"Invalid frame source '{source}' for pipeline {class_name}."
-            ) from e
+        This function will first make a lookup request to determine the frame's
+        location, which could be either on disk or inside a receiver's memory.
 
-        device = tg.DeviceProxy(rcv_url)
+        NOTE: some types of frames cannot be accessed online since they aren't buffered
+        in memory, e.g. raw frames. Such a frame will become available once the file it
+        belongs to has been closed.
 
-        getter = getattr(device, frame_source.getter_name)
+        On the other hand, some types of frames are only available from memory buffers,
+        and are never saved, e.g. "input frames". Such a frame will no longer be
+        available when the (circular) buffer that contains it has wrapped around.
 
+        Raises:
+          ValueError: Invalid frame source.
+          RuntimeError: The frame can't be retrieved.
+        """
         try:
-            raw_data: tuple[str, bytes] = getter(frame_idx)
-        except tg.DevFailed as e:
-            logger.error(
-                f"Failed to get {source} {frame_idx} (resolved to {fidx} on {device.dev_name()})"
+            lookup_result = self.session.get(
+                f"/pipeline/{str(uuid)}/lookup/{source}/{frame_idx}"
+            ).json()
+        except Lima2ValueError as e:
+            raise ValueError(f"{e}") from None
+        except Lima2LookupError as e:
+            raise RuntimeError(f"{e}") from None
+
+        h5_loc = lookup_result["hdf5"]
+        buffer_loc = lookup_result["tango"]
+
+        if h5_loc is not None:
+            try:
+                return _utils.hdf5_get_frame(
+                    url=h5_loc["url"],
+                    frame_idx=frame_idx,
+                    lookup_idx=h5_loc["idx"],
+                    frame_type=FrameType[h5_loc["frame_type"]],
+                    shape=tuple(h5_loc["shape"]),
+                )
+            except OSError as e:
+                # We cannot be too specific with the exception type here,
+                # see https://docs.h5py.org/en/stable/faq.html#exceptions.
+
+                # The file is likely still open for writing.
+                # Absorb the IO error and fall through to try online access.
+                logger.debug(
+                    f"Can't read frame from HDF5 file {h5_loc} (error: {e}) "
+                    f"-> falling back to tango access"
+                )
+
+        if buffer_loc is not None:
+            return _utils.tango_get_frame(
+                url=buffer_loc["url"],
+                command_name=buffer_loc["command_name"],
+                lookup_idx=buffer_loc["idx"],
             )
-            raise RuntimeError(
-                f"Unable to get frame {fidx} from {device.dev_name()}:\n{e}"
-            ) from e
-
-        decoder = types.decoder_by_type[frame_source.frame_type]
 
-        frm = decoder(raw_data)
-
-        return frm
+        # NOTE: if the frame source isn't buffered AND it hasn't been saved to disk,
+        # we end up with no way to fetch it.
+        raise RuntimeError(
+            f"Requested {source} {frame_idx} currently has no known location "
+            f"in either memory or disk."
+        )
 
 
 class ConnectionState(Enum):

lima2_client-4.2.0/src/lima2/common/devencoded/common.py

@@ -0,0 +1,26 @@
+# This file is part of the Lima2 project
+#
+# Copyright (c) 2020-2024 Beamline Control Unit, ESRF
+# Distributed under the MIT license. See LICENSE for more info.
+
+"""Definitions used by multiple DevEncoded deserialization modules."""
+
+from dataclasses import dataclass
+from typing import Any
+
+import numpy.typing as npt
+
+DevEncoded = bytes | tuple[str, bytes]
+
+
+@dataclass
+class Frame:
+    """
+    Provide data frame from Lima including few metadata
+    """
+
+    data: npt.NDArray[Any]
+    """Data of the frame"""
+
+    idx: int
+    """Index of the frame where 0 is the first frame"""

lima2_client-4.2.0/src/lima2/common/devencoded/data_array.py

@@ -0,0 +1,208 @@
+# This file is part of the Lima2 project
+#
+# Copyright (c) 2020-2024 Beamline Control Unit, ESRF
+# Distributed under the MIT license. See LICENSE for more info.
+
+"""Decoding of DevEncoded data in the DATA_ARRAY format.
+
+This includes multiple categories of payloads, for instance dense frames (2D) and
+vectors (1D).
+"""
+
+import struct
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Any, TypeVar
+
+import numpy as np
+import numpy.typing as npt
+
+from lima2.common.devencoded.common import DevEncoded, Frame
+from lima2.common.devencoded.exception import DevEncodedFormatNotSupported
+
+T = TypeVar("T", bound=np.generic)
+
+
+class DataArrayCategory(IntEnum):
+    """Analogue to lima::tango::category_enum. Values must match."""
+
+    SCALAR_STACK = 0
+    SPECTRUM = 1
+    IMAGE = 2
+    SPECTRUM_STACK = 3
+    IMAGE_STACK = 4
+
+
+class DataArrayDType(IntEnum):
+    """Analogue to lima::tango::data_type_enum. Values must match."""
+
+    UINT8 = 0
+    UINT16 = 1
+    UINT32 = 2
+    UINT64 = 3
+    INT8 = 4
+    INT16 = 5
+    INT32 = 6
+    INT64 = 7
+    FLOAT32 = 8
+    FLOAT64 = 9
+
+
+# Mapping used for direct conversion from raw data to numpy array
+DTYPE_TO_NUMPY: dict[DataArrayDType, type] = {
+    DataArrayDType.UINT8: np.uint8,
+    DataArrayDType.UINT16: np.uint16,
+    DataArrayDType.UINT32: np.uint32,
+    DataArrayDType.UINT64: np.uint64,
+    DataArrayDType.INT8: np.int8,
+    DataArrayDType.INT16: np.int16,
+    DataArrayDType.INT32: np.int32,
+    DataArrayDType.INT64: np.int64,
+    DataArrayDType.FLOAT32: np.float32,
+    DataArrayDType.FLOAT64: np.float64,
+}
+
+HEADER_FORMAT = "<IHHIIHHHHHHHHIIIIIIQ"
+MAGIC = struct.unpack(">I", b"DTAY")[0]
+HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
+
+
+@dataclass
+class DataArrayHeader:
+    magic: int
+    version: int
+    header_size: int
+    category: int
+    data_type: int
+    endianness: int
+    nb_dim: int
+    dim1: int
+    dim2: int
+    dim3: int
+    dim4: int
+    dim5: int
+    dim6: int
+    dim_step1: int
+    dim_step2: int
+    dim_step3: int
+    dim_step4: int
+    dim_step5: int
+    dim_step6: int
+    frame_idx: int
+
+    @staticmethod
+    def from_bytes(raw: bytes) -> "DataArrayHeader":
+        return DataArrayHeader(*struct.unpack_from(HEADER_FORMAT, raw))
+
+
+def unpack_frame(
+    payload: bytes, shape: tuple[int, int, int], dtype: np.dtype[T], frame_idx: int
+) -> Frame:
+    """Instantiate a Frame from a data array header and payload."""
+
+    data = np.frombuffer(payload, dtype=dtype).reshape(shape)
+
+    # Create a memory copy only if it is needed
+    if not data.flags.writeable:
+        data = np.array(data)
+
+    return Frame(data=data, idx=frame_idx)
+
+
+@dataclass
+class Vector:
+    """1D data, aka spectrum."""
+
+    data: npt.NDArray[Any]
+    idx: int
+
+
+def unpack_vector(payload: bytes, dtype: np.dtype[T], frame_idx: int) -> Vector:
+    """Instantiate a Vector from a data array header and payload."""
+
+    data = np.frombuffer(payload, dtype=dtype)
+
+    # Create a memory copy only if it is needed
+    if not data.flags.writeable:
+        data = np.array(data)
+
+    return Vector(data=data, idx=frame_idx)
+
+
+def decode(raw_data: DevEncoded) -> Frame | Vector:
+    """Decode a DevEncoded DATA_ARRAY and unpack the payload.
+
+    The resulting data depends on the header's category field.
+    """
+
+    if isinstance(raw_data, tuple):
+        # Support the direct output from proxy.readImage
+        if raw_data[0] not in ["DATA_ARRAY"]:
+            raise DevEncodedFormatNotSupported(
+                "Data type DATA_ARRAY expected (found %s)." % raw_data[0]
+            )
+        raw_data = raw_data[1]
+
+    header = DataArrayHeader.from_bytes(raw=raw_data)
+
+    if header.magic != MAGIC:
+        raise DevEncodedFormatNotSupported(
+            "Magic header not supported (found 0x%x)." % header.magic
+        )
+
+    # Assume backward-compatible incremental versioning
+    if header.version < 1:
+        raise DevEncodedFormatNotSupported(
+            f"Data array header version {header.version} not supported (expected >= 1)."
+        )
+
+    if header.header_size != HEADER_SIZE:
+        raise DevEncodedFormatNotSupported(
+            f"Invalid header size {header.header_size} " f"(expected {HEADER_SIZE})."
+        )
+
+    if header.endianness != 0:
+        raise DevEncodedFormatNotSupported(
+            "Unsupported endianness (found %s)." % header.endianness
+        )
+
+    try:
+        data_type = DataArrayDType(header.data_type)
+    except ValueError:
+        raise DevEncodedFormatNotSupported(
+            f"Data array data_type {header.data_type} not supported"
+        ) from None
+
+    if data_type not in DTYPE_TO_NUMPY:
+        raise NotImplementedError(
+            f"Missing dtype association for {data_type}"
+        ) from None
+
+    dtype = DTYPE_TO_NUMPY[data_type]
+
+    match header.category:
+        case DataArrayCategory.IMAGE:
+            if header.nb_dim != 3:
+                raise DevEncodedFormatNotSupported(
+                    f"Expected frame nb_dim == 3, got {header.nb_dim}"
+                )
+            return unpack_frame(
+                payload=raw_data[header.header_size :],
+                shape=(header.dim3, header.dim2, header.dim1),
+                dtype=np.dtype(dtype),
+                frame_idx=header.frame_idx,
+            )
+        case DataArrayCategory.SPECTRUM:
+            if header.nb_dim != 1:
+                raise DevEncodedFormatNotSupported(
+                    f"Expected vector nb_dim == 1, got {header.nb_dim}"
+                )
+            return unpack_vector(
+                payload=raw_data[header.header_size :],
+                dtype=np.dtype(dtype),
+                frame_idx=header.frame_idx,
+            )
+        case _:
+            raise NotImplementedError(
+                f"No unpack function for data array category {header.category}"
+            )

{lima2_client-4.1.8 → lima2_client-4.2.0}/src/lima2/common/devencoded/smx_sparse_frame.py

@@ -12,9 +12,8 @@ from typing import Any
 import numpy as np
 import numpy.typing as npt
 
-from lima2.common.devencoded.dense_frame import EncodedFrame, Frame
-
-from .exception import DevEncodedFormatNotSupported
+from lima2.common.devencoded.common import DevEncoded, Frame
+from lima2.common.devencoded.exception import DevEncodedFormatNotSupported
 
 DATA_HEADER_FORMAT = "<HHIII"
 DATA_HEADER_SIZE = struct.calcsize(DATA_HEADER_FORMAT)
@@ -91,7 +90,7 @@ class SmxSparseFrame:
         return self.densify_peaks()
 
 
-def decode(raw_data: EncodedFrame) -> SmxSparseFrame:
+def decode(raw_data: DevEncoded) -> SmxSparseFrame:
     """Decode data provided by Lima2
 
     Argument:

{lima2_client-4.1.8 → lima2_client-4.2.0}/src/lima2/common/devencoded/sparse_frame.py

@@ -12,9 +12,8 @@ from typing import Any
 import numpy as np
 import numpy.typing as npt
 
-from lima2.common.devencoded.dense_frame import EncodedFrame, Frame
-
-from .exception import DevEncodedFormatNotSupported
+from lima2.common.devencoded.common import Frame
+from lima2.common.devencoded.exception import DevEncodedFormatNotSupported
 
 DATA_HEADER_FORMAT = "<HHII"
 DATA_HEADER_SIZE = struct.calcsize(DATA_HEADER_FORMAT)
@@ -53,7 +52,10 @@ class SparseFrame:
         return Frame(dense, self.idx)
 
 
-def decode(raw_data: EncodedFrame) -> SparseFrame:
+EncodedSparseFrame = bytes | tuple[str, bytes]
+
+
+def decode(raw_data: EncodedSparseFrame) -> SparseFrame:
     """Decode data provided by Lima2
 
     Argument:

{lima2_client-4.1.8 → lima2_client-4.2.0}/src/lima2/common/devencoded/structured_array.py

@@ -11,7 +11,7 @@ from typing import Any
 import numpy
 import numpy.typing as npt
 
-from .exception import DevEncodedFormatNotSupported
+from lima2.common.devencoded.exception import DevEncodedFormatNotSupported
 
 DATA_HEADER_FORMAT = "<IHHIIHHHHHHHHIIIIIIQ"
 DATA_MAGIC = struct.unpack(">I", b"DTAY")[0]