mx-bluesky 1.5.1__py3-none-any.whl → 1.5.3__py3-none-any.whl

mx_bluesky/common/external_interaction/alerting/log_based_service.py

@@ -0,0 +1,57 @@
+import logging
+from typing import cast
+
+from dodal.log import LOGGER
+from dodal.utils import get_beamline_name
+
+from mx_bluesky.common.external_interaction.alerting import Metadata
+from mx_bluesky.common.external_interaction.alerting._service import (
+    ExtraMetadata,
+    graylog_url,
+    ispyb_url,
+)
+from mx_bluesky.common.external_interaction.callbacks.common.ispyb_mapping import (
+    get_proposal_and_session_from_visit_string,
+)
+
+
+class LoggingAlertService:
+    """
+    Implement an alert service that raises alerts by generating a specially formatted
+    log message, that may be intercepted by a logging service such as graylog and
+    used to dispatch the alert.
+    """
+
+    def __init__(self, graylog_stream_id: str, level=logging.WARNING):
+        """
+        Create a new instance of the service
+        Args:
+            level: The python logging level at which to generate the message
+        """
+        super().__init__()
+        self._level = level
+        self._graylog_stream_id = graylog_stream_id
+
+    def _append_extra_metadata(self, metadata: dict[Metadata, str]) -> dict[str, str]:
+        with_extras = cast(dict, metadata.copy())
+        with_extras[ExtraMetadata.GRAYLOG_URL] = graylog_url(self._graylog_stream_id)
+        with_extras[ExtraMetadata.BEAMLINE] = get_beamline_name("")
+        if sample_id := metadata.get(Metadata.SAMPLE_ID, None):
+            with_extras[ExtraMetadata.ISPYB_URL] = ispyb_url(sample_id)
+        if visit := metadata.get(Metadata.VISIT, None):
+            proposal, _ = get_proposal_and_session_from_visit_string(visit)
+            with_extras[ExtraMetadata.PROPOSAL] = proposal
+        return with_extras
+
+    def raise_alert(self, summary: str, content: str, metadata: dict[Metadata, str]):
+        message = f"***ALERT*** summary={summary} content={content}"
+        with_extras = self._append_extra_metadata(metadata)
+        LOGGER.log(
+            self._level,
+            message,
+            extra={
+                ExtraMetadata.ALERT_SUMMARY: summary,
+                ExtraMetadata.ALERT_CONTENT: content,
+            }
+            | with_extras,
+        )

mx_bluesky/common/external_interaction/callbacks/sample_handling/sample_handling_callback.py

@@ -1,5 +1,10 @@
+from dodal.utils import get_beamline_name
 from event_model import RunStart, RunStop
 
+from mx_bluesky.common.external_interaction.alerting import (
+    Metadata,
+    get_alerting_service,
+)
 from mx_bluesky.common.external_interaction.callbacks.common.plan_reactive_callback import (
     PlanReactiveCallback,
 )
@@ -18,30 +23,36 @@ class SampleHandlingCallback(PlanReactiveCallback):
     def __init__(self, record_loaded_on_success=False):
         super().__init__(log=ISPYB_ZOCALO_CALLBACK_LOGGER)
         self._sample_id: int | None = None
+        self._visit: str | None = None
         self._descriptor: str | None = None
         self._run_id: str | None = None
+        self._container: int | None = None
 
         # Record 'sample loaded' if document successfully stops
         self.record_loaded_on_success = record_loaded_on_success
 
     def activity_gated_start(self, doc: RunStart):
         if not self._sample_id and self.active:
-            sample_id = doc.get("metadata", {}).get("sample_id")
+            metadata = doc.get("metadata", {})
+            sample_id = metadata.get("sample_id")
             self.log.info(f"Recording sample ID at run start {sample_id}")
             self._sample_id = sample_id
+            self._visit = metadata.get("visit")
             self._run_id = self.activity_uid
+            self._container = metadata.get("container")
 
     def activity_gated_stop(self, doc: RunStop) -> RunStop:
         if self._run_id == doc.get("run_start"):
             expeye = ExpeyeInteraction()
             if doc["exit_status"] != "success":
+                reason = doc.get("reason", "")
                 exception_type, message = SampleException.type_and_message_from_reason(
-                    doc.get("reason", "")
+                    reason
                 )
                 self.log.info(
                     f"Sample handling callback intercepted exception of type {exception_type}: {message}"
                 )
-                self._record_exception(exception_type, expeye)
+                self._record_exception(exception_type, expeye, reason)
 
             elif self.record_loaded_on_success:
                 self._record_loaded(expeye)
@@ -51,10 +62,23 @@ class SampleHandlingCallback(PlanReactiveCallback):
 
         return doc
 
-    def _record_exception(self, exception_type: str, expeye: ExpeyeInteraction):
+    def _record_exception(
+        self, exception_type: str, expeye: ExpeyeInteraction, reason: str
+    ):
         assert self._sample_id, "Unable to record exception due to no sample ID"
         sample_status = self._decode_sample_status(exception_type)
         expeye.update_sample_status(self._sample_id, sample_status)
+        if sample_status == BLSampleStatus.ERROR_BEAMLINE:
+            beamline = get_beamline_name("")
+            get_alerting_service().raise_alert(
+                f"UDC encountered an error on {beamline}",
+                f"Hyperion encountered the following beamline error: {reason}",
+                {
+                    Metadata.SAMPLE_ID: str(self._sample_id),
+                    Metadata.VISIT: self._visit or "",
+                    Metadata.CONTAINER: str(self._container),
+                },
+            )
 
     def _decode_sample_status(self, exception_type: str) -> BLSampleStatus:
         match exception_type:

mx_bluesky/common/external_interaction/config_server.py

@@ -1,57 +1,154 @@
-from abc import ABC, abstractmethod
-from functools import cache
+import json
+from dataclasses import fields
+from enum import Enum
+from pathlib import Path
+from typing import Any, Generic, TypeVar
 
 from daq_config_server.client import ConfigServer
-from pydantic import BaseModel, Field, model_validator
-
-
-class FeatureFlags(BaseModel, ABC):
-    """Abstract class to use ConfigServer to toggle features for an experiment
-
-    A module wanting to use FeatureFlags should inherit this class, add boolean features
-    as attributes, and implement a get_config_server method, which returns a cached creation of
-    ConfigServer. See HyperionFeatureFlags for an example
-
-    Values supplied upon class instantiation will always take priority over the config server. If connection to the server cannot
-    be made AND values were not supplied, attributes will use their default values
-    """
-
-    # Feature values supplied at construction will override values from the config server
-    overriden_features: dict = Field(default_factory=dict, exclude=True)
-
-    @staticmethod
-    @cache
-    @abstractmethod
-    def get_config_server() -> ConfigServer: ...
-
-    @model_validator(mode="before")
-    @classmethod
-    def mark_overridden_features(cls, values):
-        assert isinstance(values, dict)
-        values["overriden_features"] = values.copy()
-        cls._validate_overridden_features(values)
-        return values
-
-    @classmethod
-    def _validate_overridden_features(cls, values: dict):
-        """Validates overridden features to ensure they are defined in the model fields."""
-        defined_fields = cls.model_fields.keys()
-        invalid_features = [key for key in values.keys() if key not in defined_fields]
-
-        if invalid_features:
-            message = f"Invalid feature toggle(s) supplied: {invalid_features}. "
-            raise ValueError(message)
-
-    def _get_flags(self):
-        flags = type(self).get_config_server().best_effort_get_all_feature_flags()
-        return {f: flags[f] for f in flags if f in self.model_fields.keys()}
-
-    def update_self_from_server(self):
-        """Used to update the feature flags from the server during a plan. Where there are flags which were explicitly set from externally supplied parameters, these values will be used instead."""
-        for flag, value in self._get_flags().items():
-            updated_value = (
-                value
-                if flag not in self.overriden_features.keys()
-                else self.overriden_features[flag]
+from pydantic import TypeAdapter
+
+from mx_bluesky.common.parameters.constants import (
+    GDA_DOMAIN_PROPERTIES_PATH,
+    FeatureSetting,
+    FeatureSettingources,
+    OavConstants,
+)
+from mx_bluesky.common.utils.log import LOGGER
+
+FEATURE_FLAG_CACHE_LENGTH_S = 60 * 5
+# Used by the config server when refreshing its cache
+_JSON_CONFIG_PATHS = [OavConstants.OAV_CONFIG_JSON]
+
+T = TypeVar("T", bound=FeatureSetting)
+
+
+class MXConfigClient(ConfigServer, Generic[T]):
+    def __init__(
+        self,
+        feature_sources: type[FeatureSettingources],
+        feature_dc: type[T],
+        url: str = "https://daq-config.diamond.ac.uk",
+    ):
+        """MX implementation of the config server client. Makes requests to the config server to retrieve config while falling back to
+        the filesystem in the case that the request failed.
+
+        See mx_bluesky/hyperion/external_interaction/config_server.py for example implementation.
+
+        Args:
+        feature_sources: A StrEnum containing available features, where the string is the name of that feature toggle in a beamline's GDA
+        domain.properties.
+
+        feature_dc: A dataclass containing available features along with their default flags. This dataclass must contain the same keys
+        as the feature_sources parameter. These defaults are used when a server request fails.
+        """
+
+        self.feature_sources = feature_sources
+        self.feature_dc: type[T] = feature_dc
+        self._cached_features: T | None = None
+        self._cached_json_config: dict[str, dict[str, Any]] = {}
+        self._time_of_last_feature_get: float = 0
+        self._verify_feature_parameters()
+        super().__init__(url)
+
+    def get_feature_flags(self) -> T:
+        """Get feature flags by making a request to the config server. If the request fails, use the hardcoded defaults. Store results in a cache
+        which should be updated at the start of a plan using self.refresh_cache()
+        """
+        return self._get_feature_flags()
+
+    def get_json_config(self, path_to_json: Path | str) -> dict[str, Any]:
+        """Get the OAV config in the form of a python dictionary. Store results in a cache
+        which should be updated at the start of a plan using self.refresh_cache().
+
+        Note that this method replicates the functionality of the config server, but provides
+        a fallback incase of server. It will be removed once the config server has been run reliably
+        for several months.
+
+        Args:
+        path_to_json: Absolute file path to json file
+        """
+        return self._get_json_config(path_to_json)
+
+    def refresh_cache(self):
+        """Refresh the client's cache. Use at the beginning of a plan"""
+        self._get_feature_flags(reset_cached_result=True)
+        for path in _JSON_CONFIG_PATHS:
+            self._get_json_config(path, reset_cached_result=True)
+
+    def _verify_feature_parameters(self):
+        sources_keys = {feature.name for feature in self.feature_sources}
+        feature_dc_keys = {key.name for key in fields(self.feature_dc)}
+        assert sources_keys == feature_dc_keys, (
+            f"MXConfig server feature_sources names do not match feature_dc keys: {sources_keys} != {feature_dc_keys}"
+        )
+
+    def _get_feature_flags(self, reset_cached_result=False) -> T:
+        """
+        Args:
+        reset_cached_result (bool): Force refresh the cache for this request
+        """
+        try:
+            if reset_cached_result:
+                self._cached_features = None
+            if not self._cached_features:
+                self._cached_features = None
+                # Construct self.feature_dc by reading the domain.properties file
+                all_features = list(self.feature_sources)
+                feature_dict = {}
+                domain_properties = self.get_file_contents(
+                    GDA_DOMAIN_PROPERTIES_PATH, reset_cached_result=reset_cached_result
+                ).splitlines()
+                for line in domain_properties:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    line = line.split("#", 1)[0].strip()  # Remove inline comments
+                    if "=" in line:
+                        key, value = map(str.strip, line.split("=", 1))
+                        for feature in all_features:
+                            assert isinstance(feature, Enum)
+                            if key == feature.value:
+                                feature_dict[feature.name] = value
+                self._check_missing_fields(
+                    {f.name for f in fields(self.feature_dc)}, set(feature_dict.keys())
+                )
+                self._cached_features = self.feature_dc(**feature_dict)
+            return self._cached_features
+        except Exception as e:
+            LOGGER.warning(
+                f"Failed to get feature flags from config server: {e} \nUsing defaults..."
+            )
+            return self.feature_dc()
+
+    def _check_missing_fields(self, expected: set, actual: set):
+        missing = expected - actual
+        if missing:
+            LOGGER.warning(
+                f"Missing features from domain.properties: {missing}.\n Using defaults for missing features"
             )
-            setattr(self, flag, updated_value)
+
+    def _get_json_config(
+        self, path_to_json: Path | str, reset_cached_result=False
+    ) -> dict[str, Any]:
+        """
+        Args:
+        reset_cached_result (bool): Force refresh the cache for this request
+        """
+        if reset_cached_result:
+            self._cached_json_config = {}
+        str_to_json = str(path_to_json)
+        if str_to_json not in self._cached_json_config:
+            try:
+                self._cached_json_config[str_to_json] = self.get_file_contents(
+                    path_to_json, dict, reset_cached_result=reset_cached_result
+                )
+
+            except Exception as e:
+                LOGGER.warning(
+                    f"Failed to get json config from config server: {e} \nReading the file directly..."
+                )
+                with open(path_to_json) as f:
+                    self._cached_json_config[str_to_json] = TypeAdapter(
+                        dict[str, Any]
+                    ).validate_python(json.loads(f.read()))
+        return self._cached_json_config[str_to_json]

mx_bluesky/common/external_interaction/ispyb/ispyb_store.py

@@ -145,11 +145,17 @@ class StoreInIspyb:
     def append_to_comment(
         self, data_collection_id: int, comment: str, delimiter: str = " "
     ) -> None:
-        with ispyb.open(self.ISPYB_CONFIG_PATH) as conn:
-            assert conn is not None, "Failed to connect to ISPyB!"
-            mx_acquisition: MXAcquisition = conn.mx_acquisition
-            mx_acquisition.update_data_collection_append_comments(
-                data_collection_id, comment, delimiter
+        try:
+            with ispyb.open(self.ISPYB_CONFIG_PATH) as conn:
+                assert conn is not None, "Failed to connect to ISPyB!"
+                mx_acquisition: MXAcquisition = conn.mx_acquisition
+                mx_acquisition.update_data_collection_append_comments(
+                    data_collection_id, comment, delimiter
+                )
+        except ispyb.ReadWriteError as e:
+            ISPYB_ZOCALO_CALLBACK_LOGGER.warning(
+                f"Unable to log comment, comment probably exceeded column length: {comment}",
+                exc_info=e,
             )
 
     def update_data_collection_group_table(
@@ -186,7 +192,6 @@ class StoreInIspyb:
             params["parentid"] = data_collection_group_id
             params["endtime"] = end_time
             params["run_status"] = run_status
-
             mx_acquisition.upsert_data_collection(list(params.values()))
 
     def _store_position_table(

mx_bluesky/common/parameters/constants.py

@@ -1,5 +1,5 @@
 import os
-from enum import Enum
+from enum import Enum, StrEnum
 
 from dodal.devices.aperturescatterguard import ApertureValue
 from dodal.devices.detector import EIGER2_X_16M_SIZE
@@ -11,6 +11,13 @@ from mx_bluesky.definitions import ROOT_DIR
 
 BEAMLINE = get_beamline_name("test")
 TEST_MODE = BEAMLINE == "test"
+ZEBRA_STATUS_TIMEOUT = 30
+
+GDA_DOMAIN_PROPERTIES_PATH = (
+    "tests/test_data/test_domain_properties"
+    if TEST_MODE
+    else (f"/dls_sw/{BEAMLINE}/software/daq_configuration/domain/domain.properties")
+)
 
 
 @dataclass(frozen=True)
@@ -27,15 +34,18 @@ class DocDescriptorNames:
     FLYSCAN_RESULTS = "flyscan_results_obtained"
 
 
+def _get_oav_config_json_path():
+    if TEST_MODE:
+        return "tests/test_data/test_OAVCentring.json"
+    elif BEAMLINE == "i03":
+        return f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
+    else:
+        return f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring.json"
+
+
 @dataclass(frozen=True)
 class OavConstants:
-    OAV_CONFIG_JSON = (
-        "tests/test_data/test_OAVCentring.json"
-        if TEST_MODE
-        else (
-            f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
-        )
-    )
+    OAV_CONFIG_JSON = _get_oav_config_json_path()
 
 
 @dataclass(frozen=True)
@@ -149,3 +159,12 @@ class Status(Enum):
     BUSY = "Busy"
     ABORTING = "Aborting"
     IDLE = "Idle"
+
+
+@dataclass
+class FeatureSetting: ...  # List of features and their default values. Subclasses must also be a pydantic dataclass
+
+
+class FeatureSettingources(
+    StrEnum
+): ...  # List of features and the name of that property in domain.properties

mx_bluesky/common/parameters/device_composites.py

@@ -11,7 +11,7 @@ from dodal.devices.fast_grid_scan import (
     ZebraFastGridScan,
 )
 from dodal.devices.flux import Flux
-from dodal.devices.i03 import Beamstop
+from dodal.devices.mx_phase1.beamstop import Beamstop
 from dodal.devices.oav.oav_detector import OAV
 from dodal.devices.oav.pin_image_recognition import PinTipDetection
 from dodal.devices.robot import BartRobot

mx_bluesky/common/parameters/gridscan.py

@@ -30,6 +30,7 @@ DETECTOR_SIZE_PER_BEAMLINE = {
     "i02-1": EIGER2_X_9M_SIZE,
     "dev": EIGER2_X_16M_SIZE,
     "i03": EIGER2_X_16M_SIZE,
+    "i04": EIGER2_X_16M_SIZE,
 }
 
 
@@ -101,7 +102,7 @@ class SpecifiedThreeDGridScan(
     """Parameters representing a so-called 3D grid scan, which consists of doing a
     gridscan in X and Y, followed by one in X and Z."""
 
-    grid1_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_1)  # type: ignore
+    grid1_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_1)
     grid2_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_2)
     x_step_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)
     y_step_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)