dls-dodal 1.64.0__py3-none-any.whl → 1.66.0__py3-none-any.whl

dodal/devices/util/lookup_tables_apple2.py

@@ -0,0 +1,390 @@
+"""Apple2 lookup table utilities and CSV converter.
+
+This module provides helpers to read, validate and convert Apple2 insertion-device
+lookup tables (energy -> gap/phase polynomials) from CSV sources into an
+in-memory dictionary format used by the Apple2 controllers.
+
+Data format produced
+The lookup-table dictionary created by convert_csv_to_lookup() follows this
+structure:
+
+{
+  "POL_MODE": {
+    "energies": {
+      "<min_energy>": {
+        "low": <float>,
+        "high": <float>,
+        "poly": <numpy.poly1d>
+      },
+      ...
+    },
+    "limit": {
+      "minimum": <float>,
+      "maximum": <float>
+    }
+  },
+}
+
+"""
+
+import csv
+import io
+from collections.abc import Generator
+from pathlib import Path
+
+import numpy as np
+from daq_config_server.client import ConfigServer
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    RootModel,
+    field_serializer,
+    field_validator,
+)
+
+from dodal.devices.apple2_undulator import Pol
+from dodal.log import LOGGER
+
+DEFAULT_GAP_FILE = "IDEnergy2GapCalibrations.csv"
+DEFAULT_PHASE_FILE = "IDEnergy2PhaseCalibrations.csv"
+
+
+DEFAULT_POLY_DEG = [
+    "7th-order",
+    "6th-order",
+    "5th-order",
+    "4th-order",
+    "3rd-order",
+    "2nd-order",
+    "1st-order",
+    "b",
+]
+
+MODE_NAME_CONVERT = {"CR": "pc", "CL": "nc"}
+
+
+class LookupTableConfig(BaseModel):
+    source: tuple[str, str] | None = None
+    mode: str = "Mode"
+    min_energy: str = "MinEnergy"
+    max_energy: str = "MaxEnergy"
+    poly_deg: list[str] = Field(default_factory=lambda: DEFAULT_POLY_DEG)
+    mode_name_convert: dict[str, str] = Field(default_factory=lambda: MODE_NAME_CONVERT)
+
+
+class EnergyMinMax(BaseModel):
+    minimum: float
+    maximum: float
+
+
+class EnergyCoverageEntry(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)  # So np.poly1d can be used.
+    low: float
+    high: float
+    poly: np.poly1d
+
+    @field_validator("poly", mode="before")
+    @classmethod
+    def validate_and_convert_poly(cls, value):
+        """If reading from serialized data, it will be using a list. Convert to np.poly1d"""
+        if isinstance(value, list):
+            return np.poly1d(value)
+        return value
+
+    @field_serializer("poly", mode="plain")
+    def serialize_poly(self, value: np.poly1d) -> list:
+        """Allow np.poly1d to work when serializing."""
+        return value.coefficients.tolist()
+
+
+class EnergyCoverage(RootModel[dict[float, EnergyCoverageEntry]]):
+    pass
+
+
+class LookupTableEntries(BaseModel):
+    energies: EnergyCoverage
+    limit: EnergyMinMax
+
+
+class LookupTable(RootModel[dict[Pol, LookupTableEntries]]):
+    # Allow to auto specify a dict if one not provided
+    def __init__(self, root: dict[Pol, LookupTableEntries] | None = None):
+        super().__init__(root=root or {})
+
+
+class GapPhaseLookupTables(BaseModel):
+    gap: LookupTable = Field(default_factory=lambda: LookupTable())
+    phase: LookupTable = Field(default_factory=lambda: LookupTable())
+
+
+def convert_csv_to_lookup(
+    file_contents: str,
+    lut_config: LookupTableConfig,
+    skip_line_start_with: str = "#",
+) -> LookupTable:
+    """
+    Convert CSV content into the Apple2 lookup-table dictionary.
+
+    Parameters:
+    -----------
+    file_contents:
+        The CSV file contents as string.
+    lut_config:
+        The configuration that how to process the file_contents into a LookupTable.
+    skip_line_start_with
+        Lines beginning with this prefix are skipped (default "#").
+
+    Returns:
+    -----------
+    LookupTable
+    """
+
+    def process_row(row: dict, lut: LookupTable):
+        """Process a single row from the CSV file and update the lookup table."""
+        mode_value = str(row[lut_config.mode]).lower()
+        if mode_value in lut_config.mode_name_convert:
+            mode_value = lut_config.mode_name_convert[f"{mode_value}"]
+        mode_value = Pol(mode_value)
+
+        # Create polynomial object for energy-to-gap/phase conversion
+        coefficients = [float(row[coef]) for coef in lut_config.poly_deg]
+        if mode_value not in lut.root:
+            lut.root[mode_value] = generate_lookup_table_entry(
+                min_energy=float(row[lut_config.min_energy]),
+                max_energy=float(row[lut_config.max_energy]),
+                poly1d_param=coefficients,
+            )
+
+        else:
+            lut.root[mode_value].energies.root[float(row[lut_config.min_energy])] = (
+                EnergyCoverageEntry(
+                    low=float(row[lut_config.min_energy]),
+                    high=float(row[lut_config.max_energy]),
+                    poly=np.poly1d(coefficients),
+                )
+            )
+
+        # Update energy limits
+        lut.root[mode_value].limit.minimum = min(
+            lut.root[mode_value].limit.minimum,
+            float(row[lut_config.min_energy]),
+        )
+        lut.root[mode_value].limit.maximum = max(
+            lut.root[mode_value].limit.maximum,
+            float(row[lut_config.max_energy]),
+        )
+        return lut
+
+    reader = csv.DictReader(read_file_and_skip(file_contents, skip_line_start_with))
+    lut = LookupTable()
+
+    for row in reader:
+        # If there are multiple source only convert requested.
+        if lut_config.source is not None:
+            if row[lut_config.source[0]] == lut_config.source[1]:
+                process_row(row=row, lut=lut)
+        else:
+            process_row(row=row, lut=lut)
+
+    # Check if our LookupTable is empty after processing, raise error if it is.
+    if not lut.root:
+        raise RuntimeError(
+            "LookupTable content is empty, failed to convert the file contents to "
+            "a LookupTable!"
+        )
+    return lut
+
+
+def read_file_and_skip(file: str, skip_line_start_with: str = "#") -> Generator[str]:
+    """Yield non-comment lines from the CSV content string."""
+    for line in io.StringIO(file):
+        if line.startswith(skip_line_start_with):
+            continue
+        else:
+            yield line
+
+
+def get_poly(
+    energy: float,
+    pol: Pol,
+    lookup_table: LookupTable,
+) -> np.poly1d:
+    """
+    Return the numpy.poly1d polynomial applicable for the given energy and polarisation.
+
+    Parameters:
+    -----------
+    energy:
+        Energy value in the same units used to create the lookup table (eV).
+    pol:
+        Polarisation mode (Pol enum).
+    lookup_table:
+        The converted lookup table dictionary for either 'gap' or 'phase'.
+    """
+    if (
+        energy < lookup_table.root[pol].limit.minimum
+        or energy > lookup_table.root[pol].limit.maximum
+    ):
+        raise ValueError(
+            "Demanding energy must lie between"
+            + f" {lookup_table.root[pol].limit.minimum}"
+            + f" and {lookup_table.root[pol].limit.maximum} eV!"
+        )
+    else:
+        for energy_range in lookup_table.root[pol].energies.root.values():
+            if energy >= energy_range.low and energy < energy_range.high:
+                return energy_range.poly
+
+    raise ValueError(
+        "Cannot find polynomial coefficients for your requested energy."
+        + " There might be gap in the calibration lookup table."
+    )
+
+
+def generate_lookup_table_entry(
+    min_energy: float, max_energy: float, poly1d_param: list[float]
+) -> LookupTableEntries:
+    return LookupTableEntries(
+        energies=EnergyCoverage(
+            {
+                min_energy: EnergyCoverageEntry(
+                    low=min_energy,
+                    high=max_energy,
+                    poly=np.poly1d(poly1d_param),
+                )
+            }
+        ),
+        limit=EnergyMinMax(
+            minimum=float(min_energy),
+            maximum=float(max_energy),
+        ),
+    )
+
+
+def generate_lookup_table(
+    pol: Pol, min_energy: float, max_energy: float, poly1d_param: list[float]
+) -> LookupTable:
+    return LookupTable(
+        {pol: generate_lookup_table_entry(min_energy, max_energy, poly1d_param)}
+    )
+
+
+def make_phase_tables(
+    pols: list[Pol],
+    min_energies: list[float],
+    max_energies: list[float],
+    poly1d_params: list[list[float]],
+) -> LookupTable:
+    """Generate a dictionary containing multiple lookuptable entries
+    for provided polarisations."""
+    lookuptable_phase = LookupTable()
+    for i in range(len(pols)):
+        lookuptable_phase.root[pols[i]] = generate_lookup_table_entry(
+            min_energy=min_energies[i],
+            max_energy=max_energies[i],
+            poly1d_param=poly1d_params[i],
+        )
+
+    return lookuptable_phase
+
+
+class EnergyMotorLookup:
+    """
+    Handles lookup tables for Apple2 ID, converting energy and polarisation to gap
+    and phase. Fetches and parses lookup tables from a config server, supports dynamic
+    updates, and validates input. If custom logic is required for lookup tables, sub
+    classes should override the _update_gap_lut and _update_phase_lut methods.
+
+    After update_lookuptable() has populated the 'gap' and 'phase' tables,
+    `get_motor_from_energy()` can be used to compute (gap, phase) for a requested
+    (energy, pol) pair.
+    """
+
+    def __init__(
+        self,
+        config_client: ConfigServer,
+        lut_config: LookupTableConfig,
+        gap_path: Path,
+        phase_path: Path,
+    ):
+        """Initialise the EnergyMotorLookup class with lookup table headers provided.
+
+        Parameters:
+        -----------
+        config_client:
+            The config server client to fetch the look up table data.
+        lut_config:
+            Configuration that defines how to process file contents into a LookupTable
+        gap_path:
+            File path to the gap lookup table.
+        phase_path:
+            File path to the phase lookup table.
+        """
+        self.lookup_tables = GapPhaseLookupTables()
+        self.config_client = config_client
+        self.lut_config = lut_config
+        self.gap_path = gap_path
+        self.phase_path = phase_path
+        self._available_pol = []
+
+    @property
+    def available_pol(self) -> list[Pol]:
+        return self._available_pol
+
+    @available_pol.setter
+    def available_pol(self, value: list[Pol]) -> None:
+        self._available_pol = value
+
+    def _update_gap_lut(self) -> None:
+        file_contents = self.config_client.get_file_contents(
+            self.gap_path, reset_cached_result=True
+        )
+        self.lookup_tables.gap = convert_csv_to_lookup(
+            file_contents, lut_config=self.lut_config
+        )
+        self.available_pol = list(self.lookup_tables.gap.root.keys())
+
+    def _update_phase_lut(self) -> None:
+        file_contents = self.config_client.get_file_contents(
+            self.phase_path, reset_cached_result=True
+        )
+        self.lookup_tables.phase = convert_csv_to_lookup(
+            file_contents, lut_config=self.lut_config
+        )
+
+    def update_lookuptables(self):
+        """
+        Update lookup tables from files and validate their format.
+        """
+        LOGGER.info("Updating lookup table from file for gap.")
+        self._update_gap_lut()
+        LOGGER.info("Updating lookup table from file for phase.")
+        self._update_phase_lut()
+
+    def get_motor_from_energy(self, energy: float, pol: Pol) -> tuple[float, float]:
+        """
+        Convert energy and polarisation to gap and phase motor positions.
+
+        Parameters:
+        -----------
+        energy : float
+            Desired energy in eV.
+        pol : Pol
+            Polarisation mode.
+
+        Returns:
+        ----------
+        tuple[float, float]
+            (gap, phase) motor positions.
+        """
+        if self.available_pol == []:
+            self.update_lookuptables()
+
+        gap_poly = get_poly(lookup_table=self.lookup_tables.gap, energy=energy, pol=pol)
+        phase_poly = get_poly(
+            lookup_table=self.lookup_tables.phase,
+            energy=energy,
+            pol=pol,
+        )
+        return gap_poly(energy), phase_poly(energy)

dodal/plan_stubs/__init__.py

@@ -0,0 +1,3 @@
+from .wrapped import move, move_relative, set_absolute, set_relative, sleep, wait
+
+__all__ = ["move", "move_relative", "set_absolute", "set_relative", "sleep", "wait"]

dodal/plans/load_panda_yaml.py

@@ -0,0 +1,9 @@
+from ophyd_async.core import YamlSettingsProvider
+from ophyd_async.fastcs.panda import HDFPanda
+from ophyd_async.plan_stubs import apply_panda_settings, retrieve_settings
+
+
+def load_panda_from_yaml(yaml_directory: str, yaml_file_name: str, panda: HDFPanda):
+    provider = YamlSettingsProvider(yaml_directory)
+    settings = yield from retrieve_settings(provider, yaml_file_name, panda)
+    yield from apply_panda_settings(settings)

dodal/plans/verify_undulator_gap.py

@@ -3,12 +3,12 @@ from typing import Protocol, runtime_checkable
 from bluesky import plan_stubs as bps
 
 from dodal.devices.common_dcm import DoubleCrystalMonochromatorBase
-from dodal.devices.undulator import Undulator
+from dodal.devices.undulator import UndulatorInKeV
 
 
 @runtime_checkable
 class CheckUndulatorDevices(Protocol):
-    undulator: Undulator
+    undulator: UndulatorInKeV
     dcm: DoubleCrystalMonochromatorBase
 
 

dodal/testing/fixtures/run_engine.py

@@ -3,29 +3,79 @@ Allow external repos to reuse these fixtures so defined in single place.
 """
 
 import asyncio
+import os
+import threading
 import time
-from collections.abc import Mapping
+from collections.abc import AsyncGenerator, Mapping
 
 import pytest
+import pytest_asyncio
+from _pytest.fixtures import FixtureRequest
 from bluesky.run_engine import RunEngine
 from bluesky.simulators import RunEngineSimulator
 
+_run_engine = RunEngine()
 
-@pytest.fixture(scope="session", autouse=True)
-async def _ensure_running_bluesky_event_loop():
-    run_engine = RunEngine()
+_ENABLE_FILEHANDLE_LEAK_CHECKS = (
+    os.getenv("DODAL_ENABLE_FILEHANDLE_LEAK_CHECKS", "").lower() == "true"
+)
+
+
+@pytest_asyncio.fixture(scope="session", loop_scope="session", autouse=True)
+async def _ensure_running_bluesky_event_loop(_global_run_engine):
     # make sure the event loop is thoroughly up and running before we try to create
     # any ophyd_async devices which might need it
     timeout = time.monotonic() + 1
-    while not run_engine.loop.is_running():
+    while not _global_run_engine.loop.is_running():
         await asyncio.sleep(0)
         if time.monotonic() > timeout:
             raise TimeoutError("This really shouldn't happen but just in case...")
 
 
 @pytest.fixture()
-async def run_engine():
-    yield RunEngine()
+async def run_engine(_global_run_engine: RunEngine) -> AsyncGenerator[RunEngine, None]:
+    try:
+        yield _global_run_engine
+    finally:
+        _global_run_engine.reset()
+
+
+@pytest_asyncio.fixture(scope="session", loop_scope="session")
+async def _global_run_engine() -> AsyncGenerator[RunEngine, None]:
+    """
+    Obtain a run engine, with its own event loop and thread.
+
+    On closure of the scope, the run engine is stopped and the event loop closed
+    in order to release all resources it consumes.
+    """
+    run_engine = RunEngine({}, call_returns_result=True)
+    yield run_engine
+    try:
+        run_engine.halt()
+    except Exception as e:
+        # Ignore exception thrown if the run engine is already halted.
+        print(f"Got exception while halting RunEngine {e}")
+    finally:
+
+        async def get_event_loop_thread():
+            """Get the thread which the run engine created for the event loop."""
+            return threading.current_thread()
+
+        fut = asyncio.run_coroutine_threadsafe(get_event_loop_thread(), run_engine.loop)
+        while not fut.done():
+            # It's not clear why this is necessary, given we are
+            # on a completely different thread and event loop
+            # but without it our future never seems to be populated with a result
+            # despite the coro getting executed
+            await asyncio.sleep(0)
+        # Terminate the event loop so that we can join() the thread
+        run_engine.loop.call_soon_threadsafe(run_engine.loop.stop)
+        run_engine_thread = fut.result()
+        run_engine_thread.join()
+        # This closes the filehandle in the event loop.
+        # This cannot be called while the event loop is running
+        run_engine.loop.close()
+    del run_engine
 
 
 @pytest.fixture
@@ -44,3 +94,25 @@ def run_engine_documents(run_engine: RunEngine) -> Mapping[str, list[dict]]:
 
     run_engine.subscribe(append_and_print)
     return docs
+
+
+@pytest.fixture(autouse=_ENABLE_FILEHANDLE_LEAK_CHECKS)
+def check_for_filehandle_leaks(request: FixtureRequest):
+    """
+    Test fixture that can be enabled in order to check for leaked filehandles
+    (typically caused by a rogue RunEngine instance).
+
+    Note that this test is not enabled by default due to imposing a significant
+    overhead. When a leak is suspected, usually from seeing a
+    PytestUnraisableExceptionWarning, enable this via autouse and run the full
+    test suite.
+    """
+    pid = os.getpid()
+    _baseline_n_open_files = len(os.listdir(f"/proc/{pid}/fd"))
+    try:
+        yield
+    finally:
+        _n_open_files = len(os.listdir(f"/proc/{pid}/fd"))
+        assert _n_open_files == _baseline_n_open_files, (
+            f"Function {request.function.__name__} leaked some filehandles"
+        )

dodal/beamline_specific_utils/i03.py

@@ -1,17 +0,0 @@
-from dataclasses import dataclass
-
-I03_BEAM_HEIGHT_UM = 20.0
-I03_BEAM_WIDTH_UM = 80.0
-
-
-@dataclass
-class BeamSize:
-    x_um: float | None
-    y_um: float | None
-
-
-def beam_size_from_aperture(aperture_size: float | None):
-    return BeamSize(
-        min(aperture_size, I03_BEAM_WIDTH_UM) if aperture_size else None,
-        I03_BEAM_HEIGHT_UM if aperture_size else None,
-    )

dodal/testing/__init__.py

@@ -1,3 +0,0 @@
-from .setup import patch_all_motors, patch_motor
-
-__all__ = ["patch_motor", "patch_all_motors"]

dodal/testing/setup.py

@@ -1,67 +0,0 @@
-from contextlib import ExitStack
-
-from ophyd_async.core import Device
-from ophyd_async.epics.motor import Motor
-from ophyd_async.testing import (
-    callback_on_mock_put,
-    set_mock_value,
-)
-
-
-def patch_motor(
-    motor: Motor,
-    initial_position: float = 0,
-    deadband: float = 0.001,
-    velocity: float = 3,
-    max_velocity: float = 5,
-    low_limit_travel: float = float("-inf"),
-    high_limit_travel: float = float("inf"),
-):
-    """
-    Patch a mock motor with sensible default values so that it can still be used in
-    tests and plans without running into errors as default values are zero.
-
-    Parameters:
-        motor: The mock motor to set mock values with.
-        initial_position: The default initial position of the motor to be set.
-        deadband: The tolerance between readback value and demand setpoint which the
-                  motor is considered at position.
-        velocity: Requested move speed when the mock motor moves.
-        max_velocity: The maximum allowable velocity that can be set for the motor.
-        low_limit_travel: The lower limit that the motor can move to.
-        high_limit_travel: The higher limit that the motor can move to.
-    """
-    set_mock_value(motor.user_setpoint, initial_position)
-    set_mock_value(motor.user_readback, initial_position)
-    set_mock_value(motor.deadband, deadband)
-    set_mock_value(motor.motor_done_move, 1)
-    set_mock_value(motor.velocity, velocity)
-    set_mock_value(motor.max_velocity, max_velocity)
-    set_mock_value(motor.low_limit_travel, low_limit_travel)
-    set_mock_value(motor.high_limit_travel, high_limit_travel)
-    return callback_on_mock_put(
-        motor.user_setpoint,
-        lambda pos, *args, **kwargs: set_mock_value(motor.user_readback, pos),
-    )
-
-
-def patch_all_motors(parent_device: Device):
-    """
-    Check all children of a device and patch any motors with mock values.
-
-    Parameters:
-        parent_device: The device that hold motor(s) as children.
-    """
-    motors = []
-
-    def recursively_find_motors(device: Device):
-        for _, child_device in device.children():
-            if isinstance(child_device, Motor):
-                motors.append(child_device)
-            recursively_find_motors(child_device)
-
-    recursively_find_motors(parent_device)
-    motor_patch_stack = ExitStack()
-    for motor in motors:
-        motor_patch_stack.enter_context(patch_motor(motor))
-    return motor_patch_stack