mx-bluesky 1.5.2__py3-none-any.whl → 1.5.4__py3-none-any.whl

mx_bluesky/common/external_interaction/callbacks/common/grid_detection_callback.py

@@ -1,4 +1,4 @@
-from typing import TypedDict
+from typing import Generic, TypedDict, TypeVar
 
 import numpy as np
 from bluesky.callbacks import CallbackBase
@@ -7,6 +7,8 @@ from event_model.documents import Event
 
 from mx_bluesky.common.utils.log import LOGGER
 
+T = TypeVar("T", int, float)
+
 
 class GridParamUpdate(TypedDict):
     """
@@ -40,14 +42,22 @@ class GridParamUpdate(TypedDict):
     z_step_size_um: float
 
 
+class XYZParams(TypedDict, Generic[T]):
+    x: T
+    y: T
+    z: T
+
+
 class GridDetectionCallback(CallbackBase):
+    OMEGA_TOLERANCE = 1
+
     def __init__(
         self,
         *args,
     ) -> None:
         super().__init__(*args)
-        self.start_positions_mm: list = []
-        self.box_numbers: list = []
+        self.start_positions_um: XYZParams[float] = XYZParams(x=0, y=0, z=0)
+        self.box_numbers: XYZParams[int] = XYZParams(x=0, y=0, z=0)
 
     def event(self, doc: Event):
         data = doc.get("data")
@@ -82,13 +92,21 @@ class GridDetectionCallback(CallbackBase):
         )
         LOGGER.info(f"Calculated start position {position_grid_start_mm}")
 
-        self.start_positions_mm.append(position_grid_start_mm)
-        self.box_numbers.append(
-            (
-                data["oav-grid_snapshot-num_boxes_x"],
-                data["oav-grid_snapshot-num_boxes_y"],
+        # If data is taken at omega=~0 then it gives us x-y info, at omega=~-90 it is x-z
+        if abs(smargon_omega) < self.OMEGA_TOLERANCE:
+            self.start_positions_um["x"] = position_grid_start_mm[0] * 1000
+            self.start_positions_um["y"] = position_grid_start_mm[1] * 1000
+            self.box_numbers["x"] = data["oav-grid_snapshot-num_boxes_x"]
+            self.box_numbers["y"] = data["oav-grid_snapshot-num_boxes_y"]
+        elif abs(smargon_omega + 90) < self.OMEGA_TOLERANCE:
+            self.start_positions_um["x"] = position_grid_start_mm[0] * 1000
+            self.start_positions_um["z"] = position_grid_start_mm[2] * 1000
+            self.box_numbers["x"] = data["oav-grid_snapshot-num_boxes_x"]
+            self.box_numbers["z"] = data["oav-grid_snapshot-num_boxes_y"]
+        else:
+            raise ValueError(
+                f"Grid detection only works at omegas of 0 or -90, omega of {smargon_omega} given."
             )
-        )
 
         self.x_step_size_um = box_width_px * microns_per_pixel_x
         self.y_step_size_um = box_width_px * microns_per_pixel_y
@@ -97,14 +115,14 @@ class GridDetectionCallback(CallbackBase):
 
     def get_grid_parameters(self) -> GridParamUpdate:
         return {
-            "x_start_um": self.start_positions_mm[0][0] * 1000,
-            "y_start_um": self.start_positions_mm[0][1] * 1000,
-            "y2_start_um": self.start_positions_mm[0][1] * 1000,
-            "z_start_um": self.start_positions_mm[1][2] * 1000,
-            "z2_start_um": self.start_positions_mm[1][2] * 1000,
-            "x_steps": self.box_numbers[0][0],
-            "y_steps": self.box_numbers[0][1],
-            "z_steps": self.box_numbers[1][1],
+            "x_start_um": self.start_positions_um["x"],
+            "y_start_um": self.start_positions_um["y"],
+            "y2_start_um": self.start_positions_um["y"],
+            "z_start_um": self.start_positions_um["z"],
+            "z2_start_um": self.start_positions_um["z"],
+            "x_steps": self.box_numbers["x"],
+            "y_steps": self.box_numbers["y"],
+            "z_steps": self.box_numbers["z"],
             "x_step_size_um": self.x_step_size_um,
             "y_step_size_um": self.y_step_size_um,
             "z_step_size_um": self.z_step_size_um,

mx_bluesky/common/external_interaction/callbacks/sample_handling/sample_handling_callback.py

@@ -1,5 +1,10 @@
+from dodal.utils import get_beamline_name
 from event_model import RunStart, RunStop
 
+from mx_bluesky.common.external_interaction.alerting import (
+    Metadata,
+    get_alerting_service,
+)
 from mx_bluesky.common.external_interaction.callbacks.common.plan_reactive_callback import (
     PlanReactiveCallback,
 )
@@ -12,36 +17,43 @@ from mx_bluesky.common.utils.log import ISPYB_ZOCALO_CALLBACK_LOGGER
 
 
 class SampleHandlingCallback(PlanReactiveCallback):
-    """Intercepts exceptions from experiment plans and updates the ISPyB BLSampleStatus
-    field according to the type of exception raised."""
+    """Intercepts exceptions from experiment plans and:
+    * Updates the ISPyB BLSampleStatus field according to the type of exception raised.
+    * Triggers an alert with details of the error."""
 
     def __init__(self, record_loaded_on_success=False):
         super().__init__(log=ISPYB_ZOCALO_CALLBACK_LOGGER)
         self._sample_id: int | None = None
+        self._visit: str | None = None
         self._descriptor: str | None = None
         self._run_id: str | None = None
+        self._container: int | None = None
 
         # Record 'sample loaded' if document successfully stops
         self.record_loaded_on_success = record_loaded_on_success
 
     def activity_gated_start(self, doc: RunStart):
         if not self._sample_id and self.active:
-            sample_id = doc.get("metadata", {}).get("sample_id")
+            metadata = doc.get("metadata", {})
+            sample_id = metadata.get("sample_id")
             self.log.info(f"Recording sample ID at run start {sample_id}")
             self._sample_id = sample_id
+            self._visit = metadata.get("visit")
             self._run_id = self.activity_uid
+            self._container = metadata.get("container")
 
     def activity_gated_stop(self, doc: RunStop) -> RunStop:
         if self._run_id == doc.get("run_start"):
             expeye = ExpeyeInteraction()
             if doc["exit_status"] != "success":
+                reason = doc.get("reason", "")
                 exception_type, message = SampleException.type_and_message_from_reason(
-                    doc.get("reason", "")
+                    reason
                 )
                 self.log.info(
                     f"Sample handling callback intercepted exception of type {exception_type}: {message}"
                 )
-                self._record_exception(exception_type, expeye)
+                self._record_exception(exception_type, expeye, reason)
 
             elif self.record_loaded_on_success:
                 self._record_loaded(expeye)
@@ -51,10 +63,23 @@ class SampleHandlingCallback(PlanReactiveCallback):
 
         return doc
 
-    def _record_exception(self, exception_type: str, expeye: ExpeyeInteraction):
+    def _record_exception(
+        self, exception_type: str, expeye: ExpeyeInteraction, reason: str
+    ):
         assert self._sample_id, "Unable to record exception due to no sample ID"
         sample_status = self._decode_sample_status(exception_type)
         expeye.update_sample_status(self._sample_id, sample_status)
+        if sample_status == BLSampleStatus.ERROR_BEAMLINE:
+            beamline = get_beamline_name("")
+            get_alerting_service().raise_alert(
+                f"UDC encountered an error on {beamline}",
+                f"Hyperion encountered the following beamline error: {reason}",
+                {
+                    Metadata.SAMPLE_ID: str(self._sample_id),
+                    Metadata.VISIT: self._visit or "",
+                    Metadata.CONTAINER: str(self._container),
+                },
+            )
 
     def _decode_sample_status(self, exception_type: str) -> BLSampleStatus:
         match exception_type:

mx_bluesky/common/external_interaction/config_server.py

@@ -1,57 +1,154 @@
-from abc import ABC, abstractmethod
-from functools import cache
+import json
+from dataclasses import fields
+from enum import Enum
+from pathlib import Path
+from typing import Any, Generic, TypeVar
 
 from daq_config_server.client import ConfigServer
-from pydantic import BaseModel, Field, model_validator
-
-
-class FeatureFlags(BaseModel, ABC):
-    """Abstract class to use ConfigServer to toggle features for an experiment
-
-    A module wanting to use FeatureFlags should inherit this class, add boolean features
-    as attributes, and implement a get_config_server method, which returns a cached creation of
-    ConfigServer. See HyperionFeatureFlags for an example
-
-    Values supplied upon class instantiation will always take priority over the config server. If connection to the server cannot
-    be made AND values were not supplied, attributes will use their default values
-    """
-
-    # Feature values supplied at construction will override values from the config server
-    overriden_features: dict = Field(default_factory=dict, exclude=True)
-
-    @staticmethod
-    @cache
-    @abstractmethod
-    def get_config_server() -> ConfigServer: ...
-
-    @model_validator(mode="before")
-    @classmethod
-    def mark_overridden_features(cls, values):
-        assert isinstance(values, dict)
-        values["overriden_features"] = values.copy()
-        cls._validate_overridden_features(values)
-        return values
-
-    @classmethod
-    def _validate_overridden_features(cls, values: dict):
-        """Validates overridden features to ensure they are defined in the model fields."""
-        defined_fields = cls.model_fields.keys()
-        invalid_features = [key for key in values.keys() if key not in defined_fields]
-
-        if invalid_features:
-            message = f"Invalid feature toggle(s) supplied: {invalid_features}. "
-            raise ValueError(message)
-
-    def _get_flags(self):
-        flags = type(self).get_config_server().best_effort_get_all_feature_flags()
-        return {f: flags[f] for f in flags if f in self.model_fields.keys()}
-
-    def update_self_from_server(self):
-        """Used to update the feature flags from the server during a plan. Where there are flags which were explicitly set from externally supplied parameters, these values will be used instead."""
-        for flag, value in self._get_flags().items():
-            updated_value = (
-                value
-                if flag not in self.overriden_features.keys()
-                else self.overriden_features[flag]
+from pydantic import TypeAdapter
+
+from mx_bluesky.common.parameters.constants import (
+    GDA_DOMAIN_PROPERTIES_PATH,
+    FeatureSetting,
+    FeatureSettingources,
+    OavConstants,
+)
+from mx_bluesky.common.utils.log import LOGGER
+
+FEATURE_FLAG_CACHE_LENGTH_S = 60 * 5
+# Used by the config server when refreshing its cache
+_JSON_CONFIG_PATHS = [OavConstants.OAV_CONFIG_JSON]
+
+T = TypeVar("T", bound=FeatureSetting)
+
+
+class MXConfigClient(ConfigServer, Generic[T]):
+    def __init__(
+        self,
+        feature_sources: type[FeatureSettingources],
+        feature_dc: type[T],
+        url: str = "https://daq-config.diamond.ac.uk",
+    ):
+        """MX implementation of the config server client. Makes requests to the config server to retrieve config while falling back to
+        the filesystem in the case that the request failed.
+
+        See mx_bluesky/hyperion/external_interaction/config_server.py for example implementation.
+
+        Args:
+        feature_sources: A StrEnum containing available features, where the string is the name of that feature toggle in a beamline's GDA
+        domain.properties.
+
+        feature_dc: A dataclass containing available features along with their default flags. This dataclass must contain the same keys
+        as the feature_sources parameter. These defaults are used when a server request fails.
+        """
+
+        self.feature_sources = feature_sources
+        self.feature_dc: type[T] = feature_dc
+        self._cached_features: T | None = None
+        self._cached_json_config: dict[str, dict[str, Any]] = {}
+        self._time_of_last_feature_get: float = 0
+        self._verify_feature_parameters()
+        super().__init__(url)
+
+    def get_feature_flags(self) -> T:
+        """Get feature flags by making a request to the config server. If the request fails, use the hardcoded defaults. Store results in a cache
+        which should be updated at the start of a plan using self.refresh_cache()
+        """
+        return self._get_feature_flags()
+
+    def get_json_config(self, path_to_json: Path | str) -> dict[str, Any]:
+        """Get the OAV config in the form of a python dictionary. Store results in a cache
+        which should be updated at the start of a plan using self.refresh_cache().
+
+        Note that this method replicates the functionality of the config server, but provides
+        a fallback incase of server. It will be removed once the config server has been run reliably
+        for several months.
+
+        Args:
+        path_to_json: Absolute file path to json file
+        """
+        return self._get_json_config(path_to_json)
+
+    def refresh_cache(self):
+        """Refresh the client's cache. Use at the beginning of a plan"""
+        self._get_feature_flags(reset_cached_result=True)
+        for path in _JSON_CONFIG_PATHS:
+            self._get_json_config(path, reset_cached_result=True)
+
+    def _verify_feature_parameters(self):
+        sources_keys = {feature.name for feature in self.feature_sources}
+        feature_dc_keys = {key.name for key in fields(self.feature_dc)}
+        assert sources_keys == feature_dc_keys, (
+            f"MXConfig server feature_sources names do not match feature_dc keys: {sources_keys} != {feature_dc_keys}"
+        )
+
+    def _get_feature_flags(self, reset_cached_result=False) -> T:
+        """
+        Args:
+        reset_cached_result (bool): Force refresh the cache for this request
+        """
+        try:
+            if reset_cached_result:
+                self._cached_features = None
+            if not self._cached_features:
+                self._cached_features = None
+                # Construct self.feature_dc by reading the domain.properties file
+                all_features = list(self.feature_sources)
+                feature_dict = {}
+                domain_properties = self.get_file_contents(
+                    GDA_DOMAIN_PROPERTIES_PATH, reset_cached_result=reset_cached_result
+                ).splitlines()
+                for line in domain_properties:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    line = line.split("#", 1)[0].strip()  # Remove inline comments
+                    if "=" in line:
+                        key, value = map(str.strip, line.split("=", 1))
+                        for feature in all_features:
+                            assert isinstance(feature, Enum)
+                            if key == feature.value:
+                                feature_dict[feature.name] = value
+                self._check_missing_fields(
+                    {f.name for f in fields(self.feature_dc)}, set(feature_dict.keys())
+                )
+                self._cached_features = self.feature_dc(**feature_dict)
+            return self._cached_features
+        except Exception as e:
+            LOGGER.warning(
+                f"Failed to get feature flags from config server: {e} \nUsing defaults..."
+            )
+            return self.feature_dc()
+
+    def _check_missing_fields(self, expected: set, actual: set):
+        missing = expected - actual
+        if missing:
+            LOGGER.warning(
+                f"Missing features from domain.properties: {missing}.\n Using defaults for missing features"
             )
-            setattr(self, flag, updated_value)
+
+    def _get_json_config(
+        self, path_to_json: Path | str, reset_cached_result=False
+    ) -> dict[str, Any]:
+        """
+        Args:
+        reset_cached_result (bool): Force refresh the cache for this request
+        """
+        if reset_cached_result:
+            self._cached_json_config = {}
+        str_to_json = str(path_to_json)
+        if str_to_json not in self._cached_json_config:
+            try:
+                self._cached_json_config[str_to_json] = self.get_file_contents(
+                    path_to_json, dict, reset_cached_result=reset_cached_result
+                )
+
+            except Exception as e:
+                LOGGER.warning(
+                    f"Failed to get json config from config server: {e} \nReading the file directly..."
+                )
+                with open(path_to_json) as f:
+                    self._cached_json_config[str_to_json] = TypeAdapter(
+                        dict[str, Any]
+                    ).validate_python(json.loads(f.read()))
+        return self._cached_json_config[str_to_json]

mx_bluesky/common/parameters/constants.py

@@ -1,5 +1,5 @@
 import os
-from enum import Enum
+from enum import Enum, StrEnum
 
 from dodal.devices.aperturescatterguard import ApertureValue
 from dodal.devices.detector import EIGER2_X_16M_SIZE
@@ -13,10 +13,17 @@ BEAMLINE = get_beamline_name("test")
 TEST_MODE = BEAMLINE == "test"
 ZEBRA_STATUS_TIMEOUT = 30
 
+GDA_DOMAIN_PROPERTIES_PATH = (
+    "tests/test_data/test_domain_properties"
+    if TEST_MODE
+    else (f"/dls_sw/{BEAMLINE}/software/daq_configuration/domain/domain.properties")
+)
+
 
 @dataclass(frozen=True)
 class DocDescriptorNames:
     # Robot load/unload event descriptor
+    ROBOT_PRE_LOAD = "robot_update_pre_load"
     ROBOT_UPDATE = "robot_update"
     # For callbacks to use
     OAV_ROTATION_SNAPSHOT_TRIGGERED = "rotation_snapshot_triggered"
@@ -28,15 +35,18 @@ class DocDescriptorNames:
     FLYSCAN_RESULTS = "flyscan_results_obtained"
 
 
+def _get_oav_config_json_path():
+    if TEST_MODE:
+        return "tests/test_data/test_OAVCentring.json"
+    elif BEAMLINE == "i03":
+        return f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
+    else:
+        return f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring.json"
+
+
 @dataclass(frozen=True)
 class OavConstants:
-    OAV_CONFIG_JSON = (
-        "tests/test_data/test_OAVCentring.json"
-        if TEST_MODE
-        else (
-            f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
-        )
-    )
+    OAV_CONFIG_JSON = _get_oav_config_json_path()
 
 
 @dataclass(frozen=True)
@@ -150,3 +160,12 @@ class Status(Enum):
     BUSY = "Busy"
     ABORTING = "Aborting"
     IDLE = "Idle"
+
+
+@dataclass
+class FeatureSetting: ...  # List of features and their default values. Subclasses must also be a pydantic dataclass
+
+
+class FeatureSettingources(
+    StrEnum
+): ...  # List of features and the name of that property in domain.properties

mx_bluesky/common/parameters/gridscan.py

@@ -102,7 +102,7 @@ class SpecifiedThreeDGridScan(
     """Parameters representing a so-called 3D grid scan, which consists of doing a
     gridscan in X and Y, followed by one in X and Z."""
 
-    grid1_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_1)  # type: ignore
+    grid1_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_1)
     grid2_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_2)
     x_step_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)
     y_step_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)