mx-bluesky 1.5.2__py3-none-any.whl → 1.5.3__py3-none-any.whl

mx_bluesky/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '1.5.2'
-__version_tuple__ = version_tuple = (1, 5, 2)
+__version__ = version = '1.5.3'
+__version_tuple__ = version_tuple = (1, 5, 3)
+
+__commit_id__ = commit_id = None

mx_bluesky/beamlines/i04/__init__.py

@@ -1,6 +1,10 @@
-from mx_bluesky.beamlines.i04.experiment_plans import (
-    i04_grid_detect_then_xray_centre_plan,
+from mx_bluesky.beamlines.i04.experiment_plans.i04_grid_detect_then_xray_centre_plan import (
+    i04_grid_detect_then_xray_centre,
 )
 from mx_bluesky.beamlines.i04.thawing_plan import thaw, thaw_and_stream_to_redis
 
-__all__ = ["thaw", "thaw_and_stream_to_redis", "i04_grid_detect_then_xray_centre_plan"]
+__all__ = [
+    "thaw",
+    "thaw_and_stream_to_redis",
+    "i04_grid_detect_then_xray_centre",
+]

mx_bluesky/beamlines/i04/experiment_plans/i04_grid_detect_then_xray_centre_plan.py

@@ -67,6 +67,7 @@ from mx_bluesky.common.preprocessors.preprocessors import (
     transmission_and_xbpm_feedback_for_collection_decorator,
 )
 from mx_bluesky.common.utils.context import device_composite_from_context
+from mx_bluesky.common.utils.log import LOGGER
 from mx_bluesky.phase1_zebra.device_setup_plans.setup_zebra import (
     setup_zebra_for_gridscan,
     tidy_up_zebra_after_gridscan,
@@ -184,9 +185,11 @@ def get_ready_for_oav_and_close_shutter(
 ):
     yield from bps.wait(PlanGroupCheckpointConstants.GRID_READY_FOR_DC)
     group = "get_ready_for_oav_and_close_shutter"
+    LOGGER.info("Non-udc tidy: Seting up beamline for OAV")
     yield from setup_beamline_for_OAV(
         smargon, backlight, aperture_scatterguard, group=group
     )
+    LOGGER.info("Non-udc tidy: Closing detector shutter")
     yield from bps.abs_set(
         detector_motion.shutter,
         0,

mx_bluesky/beamlines/i24/serial/blueapi_config.yaml

@@ -7,11 +7,11 @@ env:
   events:
     broadcast_status_events: false
 api:
-  port: 25565
+  url: http://localhost:25565
   cors:
     allow_credentials: True
     origins: 
       - "*"
 stomp:
   enabled: true
-  host: i24-control.diamond.ac.uk
+  url: http://i24-control.diamond.ac.uk:61613

mx_bluesky/common/experiment_plans/oav_grid_detection_plan.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
 import bluesky.plan_stubs as bps
 import numpy as np
 from blueapi.core import BlueskyContext
+from bluesky.utils import MsgGenerator
 from dodal.devices.oav.oav_detector import OAV
 from dodal.devices.oav.pin_image_recognition import PinTipDetection
 from dodal.devices.oav.pin_image_recognition.utils import NONE_VALUE
@@ -49,6 +50,16 @@ def get_min_and_max_y_of_pin(
     return min_y, max_y
 
 
+def optimum_grid_detect_angles(smargon: Smargon) -> MsgGenerator[list[float]]:
+    """We need to match the 0 and -90 that the fast grid scan performs but the order in
+    which we do the grid detection does not matter so we do the closest angle first."""
+    current_omega = yield from bps.rd(smargon.omega)
+    if current_omega < -45:
+        return [-90, 0]
+    else:
+        return [0, -90]
+
+
 def grid_detection_plan(
     composite: OavGridDetectionComposite,
     parameters: OAVParameters,
@@ -90,8 +101,7 @@ def grid_detection_plan(
 
     grid_width_pixels = int(grid_width_microns / microns_per_pixel_x)
 
-    # The FGS uses -90 so we need to match it
-    for angle in [0, -90]:
+    for angle in (yield from optimum_grid_detect_angles(smargon)):
         yield from bps.mv(smargon.omega, angle)
         # need to wait for the OAV image to update
         # See #673 for improvements

mx_bluesky/common/external_interaction/alerting/__init__.py

@@ -0,0 +1,13 @@
+from mx_bluesky.common.external_interaction.alerting._service import (
+    AlertService,
+    Metadata,
+    get_alerting_service,
+    set_alerting_service,
+)
+
+__all__ = [
+    "AlertService",
+    "Metadata",
+    "get_alerting_service",
+    "set_alerting_service",
+]

mx_bluesky/common/external_interaction/alerting/_service.py

@@ -0,0 +1,82 @@
+from datetime import UTC, datetime, timedelta
+from enum import StrEnum
+from typing import Protocol
+from urllib.parse import quote, urlencode
+
+
+class Metadata(StrEnum):
+    """Metadata fields that can be specified by the caller when raising an alert."""
+
+    CONTAINER = "container"
+    SAMPLE_ID = "sample_id"
+    VISIT = "visit"
+
+
+class ExtraMetadata(StrEnum):
+    """Additional metadata fields that are automatically appended by
+    the AlertService implementations."""
+
+    ALERT_CONTENT = "alert_content"
+    ALERT_SUMMARY = "alert_summary"
+    BEAMLINE = "beamline"
+    GRAYLOG_URL = "graylog_url"
+    ISPYB_URL = "ispyb_url"
+    PROPOSAL = "proposal"
+
+
+class AlertService(Protocol):
+    """
+    Implemented by any backend that provides the ability to dispatch alerts to some
+    service that is capable of disseminating them via any of a variety of media such
+    as email, SMS, instant messaging, etc etc.
+    """
+
+    def raise_alert(self, summary: str, content: str, metadata: dict[Metadata, str]):
+        """
+        Raise an alert that will be forwarded to beamline support staff, which might
+        for example be used as the basis for an incident in an incident reporting system.
+        Args:
+            summary: One line summary of the alert, that might for instance be used
+                in an email subject line.
+            content: Plain text content detailing the nature of the incident.
+            metadata: A dict of strings that can be included as metadata in the alert for
+                those backends that support it. The summary and content will be included
+                by default.
+        """
+        pass
+
+
+_alert_service: AlertService
+
+
+def get_alerting_service() -> AlertService:
+    """Get the alert service for this instance."""
+    return _alert_service
+
+
+def set_alerting_service(service: AlertService):
+    """Set the alert service for this instance, call when the beamline is initialised."""
+    global _alert_service
+    _alert_service = service
+
+
+def ispyb_url(sample_id: str):
+    return f"https://ispyb.diamond.ac.uk/samples/sid/{quote(sample_id)}"
+
+
+def graylog_url(stream_id: str):
+    now = datetime.now(UTC)
+    from_utc = now - timedelta(minutes=5)
+    from_timestamp = from_utc.isoformat()
+    # Add 1 second for graylog timing jitter
+    to_utc = now + timedelta(seconds=1)
+    to_timestamp = to_utc.isoformat()
+    query_string = urlencode(
+        {
+            "streams": stream_id,
+            "rangetype": "absolute",
+            "from": from_timestamp,
+            "to": to_timestamp,
+        }
+    )
+    return "https://graylog.diamond.ac.uk/search?" + query_string

mx_bluesky/common/external_interaction/alerting/log_based_service.py

@@ -0,0 +1,57 @@
+import logging
+from typing import cast
+
+from dodal.log import LOGGER
+from dodal.utils import get_beamline_name
+
+from mx_bluesky.common.external_interaction.alerting import Metadata
+from mx_bluesky.common.external_interaction.alerting._service import (
+    ExtraMetadata,
+    graylog_url,
+    ispyb_url,
+)
+from mx_bluesky.common.external_interaction.callbacks.common.ispyb_mapping import (
+    get_proposal_and_session_from_visit_string,
+)
+
+
+class LoggingAlertService:
+    """
+    Implement an alert service that raises alerts by generating a specially formatted
+    log message, that may be intercepted by a logging service such as graylog and
+    used to dispatch the alert.
+    """
+
+    def __init__(self, graylog_stream_id: str, level=logging.WARNING):
+        """
+        Create a new instance of the service
+        Args:
+            level: The python logging level at which to generate the message
+        """
+        super().__init__()
+        self._level = level
+        self._graylog_stream_id = graylog_stream_id
+
+    def _append_extra_metadata(self, metadata: dict[Metadata, str]) -> dict[str, str]:
+        with_extras = cast(dict, metadata.copy())
+        with_extras[ExtraMetadata.GRAYLOG_URL] = graylog_url(self._graylog_stream_id)
+        with_extras[ExtraMetadata.BEAMLINE] = get_beamline_name("")
+        if sample_id := metadata.get(Metadata.SAMPLE_ID, None):
+            with_extras[ExtraMetadata.ISPYB_URL] = ispyb_url(sample_id)
+        if visit := metadata.get(Metadata.VISIT, None):
+            proposal, _ = get_proposal_and_session_from_visit_string(visit)
+            with_extras[ExtraMetadata.PROPOSAL] = proposal
+        return with_extras
+
+    def raise_alert(self, summary: str, content: str, metadata: dict[Metadata, str]):
+        message = f"***ALERT*** summary={summary} content={content}"
+        with_extras = self._append_extra_metadata(metadata)
+        LOGGER.log(
+            self._level,
+            message,
+            extra={
+                ExtraMetadata.ALERT_SUMMARY: summary,
+                ExtraMetadata.ALERT_CONTENT: content,
+            }
+            | with_extras,
+        )

mx_bluesky/common/external_interaction/callbacks/sample_handling/sample_handling_callback.py

@@ -1,5 +1,10 @@
+from dodal.utils import get_beamline_name
 from event_model import RunStart, RunStop
 
+from mx_bluesky.common.external_interaction.alerting import (
+    Metadata,
+    get_alerting_service,
+)
 from mx_bluesky.common.external_interaction.callbacks.common.plan_reactive_callback import (
     PlanReactiveCallback,
 )
@@ -18,30 +23,36 @@ class SampleHandlingCallback(PlanReactiveCallback):
     def __init__(self, record_loaded_on_success=False):
         super().__init__(log=ISPYB_ZOCALO_CALLBACK_LOGGER)
         self._sample_id: int | None = None
+        self._visit: str | None = None
         self._descriptor: str | None = None
         self._run_id: str | None = None
+        self._container: int | None = None
 
         # Record 'sample loaded' if document successfully stops
         self.record_loaded_on_success = record_loaded_on_success
 
     def activity_gated_start(self, doc: RunStart):
         if not self._sample_id and self.active:
-            sample_id = doc.get("metadata", {}).get("sample_id")
+            metadata = doc.get("metadata", {})
+            sample_id = metadata.get("sample_id")
             self.log.info(f"Recording sample ID at run start {sample_id}")
             self._sample_id = sample_id
+            self._visit = metadata.get("visit")
             self._run_id = self.activity_uid
+            self._container = metadata.get("container")
 
     def activity_gated_stop(self, doc: RunStop) -> RunStop:
         if self._run_id == doc.get("run_start"):
             expeye = ExpeyeInteraction()
             if doc["exit_status"] != "success":
+                reason = doc.get("reason", "")
                 exception_type, message = SampleException.type_and_message_from_reason(
-                    doc.get("reason", "")
+                    reason
                 )
                 self.log.info(
                     f"Sample handling callback intercepted exception of type {exception_type}: {message}"
                 )
-                self._record_exception(exception_type, expeye)
+                self._record_exception(exception_type, expeye, reason)
 
             elif self.record_loaded_on_success:
                 self._record_loaded(expeye)
@@ -51,10 +62,23 @@ class SampleHandlingCallback(PlanReactiveCallback):
 
         return doc
 
-    def _record_exception(self, exception_type: str, expeye: ExpeyeInteraction):
+    def _record_exception(
+        self, exception_type: str, expeye: ExpeyeInteraction, reason: str
+    ):
         assert self._sample_id, "Unable to record exception due to no sample ID"
         sample_status = self._decode_sample_status(exception_type)
         expeye.update_sample_status(self._sample_id, sample_status)
+        if sample_status == BLSampleStatus.ERROR_BEAMLINE:
+            beamline = get_beamline_name("")
+            get_alerting_service().raise_alert(
+                f"UDC encountered an error on {beamline}",
+                f"Hyperion encountered the following beamline error: {reason}",
+                {
+                    Metadata.SAMPLE_ID: str(self._sample_id),
+                    Metadata.VISIT: self._visit or "",
+                    Metadata.CONTAINER: str(self._container),
+                },
+            )
 
     def _decode_sample_status(self, exception_type: str) -> BLSampleStatus:
         match exception_type:

mx_bluesky/common/external_interaction/config_server.py

@@ -1,57 +1,154 @@
-from abc import ABC, abstractmethod
-from functools import cache
+import json
+from dataclasses import fields
+from enum import Enum
+from pathlib import Path
+from typing import Any, Generic, TypeVar
 
 from daq_config_server.client import ConfigServer
-from pydantic import BaseModel, Field, model_validator
-
-
-class FeatureFlags(BaseModel, ABC):
-    """Abstract class to use ConfigServer to toggle features for an experiment
-
-    A module wanting to use FeatureFlags should inherit this class, add boolean features
-    as attributes, and implement a get_config_server method, which returns a cached creation of
-    ConfigServer. See HyperionFeatureFlags for an example
-
-    Values supplied upon class instantiation will always take priority over the config server. If connection to the server cannot
-    be made AND values were not supplied, attributes will use their default values
-    """
-
-    # Feature values supplied at construction will override values from the config server
-    overriden_features: dict = Field(default_factory=dict, exclude=True)
-
-    @staticmethod
-    @cache
-    @abstractmethod
-    def get_config_server() -> ConfigServer: ...
-
-    @model_validator(mode="before")
-    @classmethod
-    def mark_overridden_features(cls, values):
-        assert isinstance(values, dict)
-        values["overriden_features"] = values.copy()
-        cls._validate_overridden_features(values)
-        return values
-
-    @classmethod
-    def _validate_overridden_features(cls, values: dict):
-        """Validates overridden features to ensure they are defined in the model fields."""
-        defined_fields = cls.model_fields.keys()
-        invalid_features = [key for key in values.keys() if key not in defined_fields]
-
-        if invalid_features:
-            message = f"Invalid feature toggle(s) supplied: {invalid_features}. "
-            raise ValueError(message)
-
-    def _get_flags(self):
-        flags = type(self).get_config_server().best_effort_get_all_feature_flags()
-        return {f: flags[f] for f in flags if f in self.model_fields.keys()}
-
-    def update_self_from_server(self):
-        """Used to update the feature flags from the server during a plan. Where there are flags which were explicitly set from externally supplied parameters, these values will be used instead."""
-        for flag, value in self._get_flags().items():
-            updated_value = (
-                value
-                if flag not in self.overriden_features.keys()
-                else self.overriden_features[flag]
+from pydantic import TypeAdapter
+
+from mx_bluesky.common.parameters.constants import (
+    GDA_DOMAIN_PROPERTIES_PATH,
+    FeatureSetting,
+    FeatureSettingources,
+    OavConstants,
+)
+from mx_bluesky.common.utils.log import LOGGER
+
+FEATURE_FLAG_CACHE_LENGTH_S = 60 * 5
+# Used by the config server when refreshing its cache
+_JSON_CONFIG_PATHS = [OavConstants.OAV_CONFIG_JSON]
+
+T = TypeVar("T", bound=FeatureSetting)
+
+
+class MXConfigClient(ConfigServer, Generic[T]):
+    def __init__(
+        self,
+        feature_sources: type[FeatureSettingources],
+        feature_dc: type[T],
+        url: str = "https://daq-config.diamond.ac.uk",
+    ):
+        """MX implementation of the config server client. Makes requests to the config server to retrieve config while falling back to
+        the filesystem in the case that the request failed.
+
+        See mx_bluesky/hyperion/external_interaction/config_server.py for example implementation.
+
+        Args:
+        feature_sources: A StrEnum containing available features, where the string is the name of that feature toggle in a beamline's GDA
+        domain.properties.
+
+        feature_dc: A dataclass containing available features along with their default flags. This dataclass must contain the same keys
+        as the feature_sources parameter. These defaults are used when a server request fails.
+        """
+
+        self.feature_sources = feature_sources
+        self.feature_dc: type[T] = feature_dc
+        self._cached_features: T | None = None
+        self._cached_json_config: dict[str, dict[str, Any]] = {}
+        self._time_of_last_feature_get: float = 0
+        self._verify_feature_parameters()
+        super().__init__(url)
+
+    def get_feature_flags(self) -> T:
+        """Get feature flags by making a request to the config server. If the request fails, use the hardcoded defaults. Store results in a cache
+        which should be updated at the start of a plan using self.refresh_cache()
+        """
+        return self._get_feature_flags()
+
+    def get_json_config(self, path_to_json: Path | str) -> dict[str, Any]:
+        """Get the OAV config in the form of a python dictionary. Store results in a cache
+        which should be updated at the start of a plan using self.refresh_cache().
+
+        Note that this method replicates the functionality of the config server, but provides
+        a fallback incase of server. It will be removed once the config server has been run reliably
+        for several months.
+
+        Args:
+        path_to_json: Absolute file path to json file
+        """
+        return self._get_json_config(path_to_json)
+
+    def refresh_cache(self):
+        """Refresh the client's cache. Use at the beginning of a plan"""
+        self._get_feature_flags(reset_cached_result=True)
+        for path in _JSON_CONFIG_PATHS:
+            self._get_json_config(path, reset_cached_result=True)
+
+    def _verify_feature_parameters(self):
+        sources_keys = {feature.name for feature in self.feature_sources}
+        feature_dc_keys = {key.name for key in fields(self.feature_dc)}
+        assert sources_keys == feature_dc_keys, (
+            f"MXConfig server feature_sources names do not match feature_dc keys: {sources_keys} != {feature_dc_keys}"
+        )
+
+    def _get_feature_flags(self, reset_cached_result=False) -> T:
+        """
+        Args:
+        reset_cached_result (bool): Force refresh the cache for this request
+        """
+        try:
+            if reset_cached_result:
+                self._cached_features = None
+            if not self._cached_features:
+                self._cached_features = None
+                # Construct self.feature_dc by reading the domain.properties file
+                all_features = list(self.feature_sources)
+                feature_dict = {}
+                domain_properties = self.get_file_contents(
+                    GDA_DOMAIN_PROPERTIES_PATH, reset_cached_result=reset_cached_result
+                ).splitlines()
+                for line in domain_properties:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    line = line.split("#", 1)[0].strip()  # Remove inline comments
+                    if "=" in line:
+                        key, value = map(str.strip, line.split("=", 1))
+                        for feature in all_features:
+                            assert isinstance(feature, Enum)
+                            if key == feature.value:
+                                feature_dict[feature.name] = value
+                self._check_missing_fields(
+                    {f.name for f in fields(self.feature_dc)}, set(feature_dict.keys())
+                )
+                self._cached_features = self.feature_dc(**feature_dict)
+            return self._cached_features
+        except Exception as e:
+            LOGGER.warning(
+                f"Failed to get feature flags from config server: {e} \nUsing defaults..."
+            )
+            return self.feature_dc()
+
+    def _check_missing_fields(self, expected: set, actual: set):
+        missing = expected - actual
+        if missing:
+            LOGGER.warning(
+                f"Missing features from domain.properties: {missing}.\n Using defaults for missing features"
             )
-            setattr(self, flag, updated_value)
+
+    def _get_json_config(
+        self, path_to_json: Path | str, reset_cached_result=False
+    ) -> dict[str, Any]:
+        """
+        Args:
+        reset_cached_result (bool): Force refresh the cache for this request
+        """
+        if reset_cached_result:
+            self._cached_json_config = {}
+        str_to_json = str(path_to_json)
+        if str_to_json not in self._cached_json_config:
+            try:
+                self._cached_json_config[str_to_json] = self.get_file_contents(
+                    path_to_json, dict, reset_cached_result=reset_cached_result
+                )
+
+            except Exception as e:
+                LOGGER.warning(
+                    f"Failed to get json config from config server: {e} \nReading the file directly..."
+                )
+                with open(path_to_json) as f:
+                    self._cached_json_config[str_to_json] = TypeAdapter(
+                        dict[str, Any]
+                    ).validate_python(json.loads(f.read()))
+        return self._cached_json_config[str_to_json]

mx_bluesky/common/parameters/constants.py

@@ -1,5 +1,5 @@
 import os
-from enum import Enum
+from enum import Enum, StrEnum
 
 from dodal.devices.aperturescatterguard import ApertureValue
 from dodal.devices.detector import EIGER2_X_16M_SIZE
@@ -13,6 +13,12 @@ BEAMLINE = get_beamline_name("test")
 TEST_MODE = BEAMLINE == "test"
 ZEBRA_STATUS_TIMEOUT = 30
 
+GDA_DOMAIN_PROPERTIES_PATH = (
+    "tests/test_data/test_domain_properties"
+    if TEST_MODE
+    else (f"/dls_sw/{BEAMLINE}/software/daq_configuration/domain/domain.properties")
+)
+
 
 @dataclass(frozen=True)
 class DocDescriptorNames:
@@ -28,15 +34,18 @@ class DocDescriptorNames:
     FLYSCAN_RESULTS = "flyscan_results_obtained"
 
 
+def _get_oav_config_json_path():
+    if TEST_MODE:
+        return "tests/test_data/test_OAVCentring.json"
+    elif BEAMLINE == "i03":
+        return f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
+    else:
+        return f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring.json"
+
+
 @dataclass(frozen=True)
 class OavConstants:
-    OAV_CONFIG_JSON = (
-        "tests/test_data/test_OAVCentring.json"
-        if TEST_MODE
-        else (
-            f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
-        )
-    )
+    OAV_CONFIG_JSON = _get_oav_config_json_path()
 
 
 @dataclass(frozen=True)
@@ -150,3 +159,12 @@ class Status(Enum):
     BUSY = "Busy"
     ABORTING = "Aborting"
     IDLE = "Idle"
+
+
+@dataclass
+class FeatureSetting: ...  # List of features and their default values. Subclasses must also be a pydantic dataclass
+
+
+class FeatureSettingources(
+    StrEnum
+): ...  # List of features and the name of that property in domain.properties

mx_bluesky/common/parameters/gridscan.py

@@ -102,7 +102,7 @@ class SpecifiedThreeDGridScan(
     """Parameters representing a so-called 3D grid scan, which consists of doing a
     gridscan in X and Y, followed by one in X and Z."""
 
-    grid1_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_1)  # type: ignore
+    grid1_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_1)
     grid2_omega_deg: float = Field(default=GridscanParamConstants.OMEGA_2)
     x_step_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)
     y_step_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)