mx-bluesky 1.1.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

mx_bluesky/hyperion/external_interaction/callbacks/grid_detection_callback.py

@@ -2,7 +2,6 @@ from typing import TypedDict
 
 import numpy as np
 from bluesky.callbacks import CallbackBase
-from dodal.devices.oav.oav_detector import OAVConfigParams
 from dodal.devices.oav.utils import calculate_x_y_z_of_pixel
 from event_model.documents import Event
 
@@ -26,21 +25,19 @@ class GridParamUpdate(TypedDict):
 class GridDetectionCallback(CallbackBase):
     def __init__(
         self,
-        oav_params: OAVConfigParams,
         *args,
     ) -> None:
         super().__init__(*args)
-        self.oav_params = oav_params
         self.start_positions: list = []
         self.box_numbers: list = []
 
     def event(self, doc: Event):
         data = doc.get("data")
-        top_left_x_px = data["oav_grid_snapshot_top_left_x"]
-        box_width_px = data["oav_grid_snapshot_box_width"]
+        top_left_x_px = data["oav-grid_snapshot-top_left_x"]
+        box_width_px = data["oav-grid_snapshot-box_width"]
         x_of_centre_of_first_box_px = top_left_x_px + box_width_px / 2
 
-        top_left_y_px = data["oav_grid_snapshot_top_left_y"]
+        top_left_y_px = data["oav-grid_snapshot-top_left_y"]
         y_of_centre_of_first_box_px = top_left_y_px + box_width_px / 2
 
         smargon_omega = data["smargon-omega"]
@@ -53,23 +50,31 @@ class GridDetectionCallback(CallbackBase):
             y_of_centre_of_first_box_px,
         )
 
+        microns_per_pixel_x = data["oav-microns_per_pixel_x"]
+        microns_per_pixel_y = data["oav-microns_per_pixel_y"]
+        beam_x = data["oav-beam_centre_i"]
+        beam_y = data["oav-beam_centre_j"]
+
         position_grid_start = calculate_x_y_z_of_pixel(
-            current_xyz, smargon_omega, centre_of_first_box, self.oav_params
+            current_xyz,
+            smargon_omega,
+            centre_of_first_box,
+            (beam_x, beam_y),
+            (microns_per_pixel_x, microns_per_pixel_y),
         )
-
         LOGGER.info(f"Calculated start position {position_grid_start}")
 
         self.start_positions.append(position_grid_start)
         self.box_numbers.append(
             (
-                data["oav_grid_snapshot_num_boxes_x"],
-                data["oav_grid_snapshot_num_boxes_y"],
+                data["oav-grid_snapshot-num_boxes_x"],
+                data["oav-grid_snapshot-num_boxes_y"],
             )
         )
 
-        self.x_step_size_mm = box_width_px * self.oav_params.micronsPerXPixel / 1000
-        self.y_step_size_mm = box_width_px * self.oav_params.micronsPerYPixel / 1000
-        self.z_step_size_mm = box_width_px * self.oav_params.micronsPerYPixel / 1000
+        self.x_step_size_mm = box_width_px * microns_per_pixel_x / 1000
+        self.y_step_size_mm = box_width_px * microns_per_pixel_y / 1000
+        self.z_step_size_mm = box_width_px * microns_per_pixel_y / 1000
         return doc
 
     def get_grid_parameters(self) -> GridParamUpdate:

mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py

@@ -5,9 +5,12 @@ from collections.abc import Callable, Sequence
 from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from dodal.beamline_specific_utils.i03 import beam_size_from_aperture
+from dodal.devices.detector import DetectorParams
 from dodal.devices.detector.det_resolution import resolution
 from dodal.devices.synchrotron import SynchrotronMode
 
+from mx_bluesky.common.parameters.components import DiffractionExperimentWithSample
+from mx_bluesky.common.utils.log import set_dcgid_tag
 from mx_bluesky.hyperion.external_interaction.callbacks.plan_reactive_callback import (
     PlanReactiveCallback,
 )
@@ -21,8 +24,7 @@ from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_store import (
     StoreInIspyb,
 )
 from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_utils import get_ispyb_config
-from mx_bluesky.hyperion.log import ISPYB_LOGGER, set_dcgid_tag
-from mx_bluesky.hyperion.parameters.components import DiffractionExperimentWithSample
+from mx_bluesky.hyperion.log import ISPYB_LOGGER
 from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.utils.utils import convert_eV_to_angstrom
 
@@ -33,6 +35,25 @@ if TYPE_CHECKING:
     from event_model.documents import Event, EventDescriptor, RunStart, RunStop
 
 
+def _update_based_on_energy(
+    doc: Event,
+    detector_params: DetectorParams,
+    data_collection_info: DataCollectionInfo,
+):
+    """If energy has been read as part of this reading then add it into the data
+    collection info along with the other fields that depend on it."""
+    if energy_kev := doc["data"].get("dcm-energy_in_kev", None):
+        energy_ev = energy_kev * 1000
+        wavelength_angstroms = convert_eV_to_angstrom(energy_ev)
+        data_collection_info.wavelength = wavelength_angstroms
+        data_collection_info.resolution = resolution(
+            detector_params,
+            wavelength_angstroms,
+            detector_params.detector_distance,
+        )
+    return data_collection_info
+
+
 class BaseISPyBCallback(PlanReactiveCallback):
     def __init__(
         self,
@@ -109,10 +130,13 @@ class BaseISPyBCallback(PlanReactiveCallback):
             slitgap_horizontal=doc["data"]["s4_slit_gaps_xgap"],
             slitgap_vertical=doc["data"]["s4_slit_gaps_ygap"],
         )
+        hwscan_data_collection_info = _update_based_on_energy(
+            doc, self.params.detector_params, hwscan_data_collection_info
+        )
         hwscan_position_info = DataCollectionPositionInfo(
-            pos_x=doc["data"]["smargon-x"],
-            pos_y=doc["data"]["smargon-y"],
-            pos_z=doc["data"]["smargon-z"],
+            pos_x=float(doc["data"]["smargon-x"]),
+            pos_y=float(doc["data"]["smargon-y"]),
+            pos_z=float(doc["data"]["smargon-z"]),
         )
         scan_data_infos = self.populate_info_for_update(
             hwscan_data_collection_info, hwscan_position_info, self.params
@@ -137,16 +161,9 @@ class BaseISPyBCallback(PlanReactiveCallback):
         if transmission := doc["data"]["attenuator-actual_transmission"]:
             # Ispyb wants the transmission in a percentage, we use fractions
             hwscan_data_collection_info.transmission = transmission * 100
-        event_energy = doc["data"]["dcm-energy_in_kev"]
-        if event_energy:
-            energy_ev = event_energy * 1000
-            wavelength_angstroms = convert_eV_to_angstrom(energy_ev)
-            hwscan_data_collection_info.wavelength = wavelength_angstroms
-            hwscan_data_collection_info.resolution = resolution(
-                self.params.detector_params,
-                wavelength_angstroms,
-                self.params.detector_params.detector_distance,
-            )
+        hwscan_data_collection_info = _update_based_on_energy(
+            doc, self.params.detector_params, hwscan_data_collection_info
+        )
         scan_data_infos = self.populate_info_for_update(
             hwscan_data_collection_info, None, self.params
         )

mx_bluesky/hyperion/external_interaction/callbacks/log_uid_tag_callback.py

@@ -1,7 +1,7 @@
 from bluesky.callbacks import CallbackBase
 from event_model import RunStart, RunStop
 
-from mx_bluesky.hyperion.log import set_uid_tag
+from mx_bluesky.common.utils.log import set_uid_tag
 
 
 class LogUidTaggingCallback(CallbackBase):

mx_bluesky/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py

@@ -6,7 +6,6 @@ from event_model.documents import EventDescriptor
 
 from mx_bluesky.hyperion.external_interaction.callbacks.common.ispyb_mapping import (
     get_proposal_and_session_from_visit_string,
-    get_visit_string_from_path,
 )
 from mx_bluesky.hyperion.external_interaction.callbacks.plan_reactive_callback import (
     PlanReactiveCallback,
@@ -37,10 +36,9 @@ class RobotLoadISPyBCallback(PlanReactiveCallback):
             ISPYB_LOGGER.debug(f"ISPyB robot load callback received: {doc}")
             self.run_uid = doc.get("uid")
             assert isinstance(metadata := doc.get("metadata"), dict)
-            assert isinstance(
-                visit := get_visit_string_from_path(metadata["visit_path"]), str
+            proposal, session = get_proposal_and_session_from_visit_string(
+                metadata["visit"]
             )
-            proposal, session = get_proposal_and_session_from_visit_string(visit)
             self.action_id = self.expeye.start_load(
                 proposal,
                 session,
@@ -64,7 +62,7 @@ class RobotLoadISPyBCallback(PlanReactiveCallback):
                 self.action_id is not None
             ), "ISPyB Robot load callback event called unexpectedly"
             barcode = doc["data"]["robot-barcode"]
-            oav_snapshot = doc["data"]["oav_snapshot_last_saved_path"]
+            oav_snapshot = doc["data"]["oav-snapshot-last_saved_path"]
             webcam_snapshot = doc["data"]["webcam-last_saved_path"]
             # I03 uses webcam/oav snapshots in place of before/after snapshots
             self.expeye.update_barcode_and_snapshots(

mx_bluesky/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 from collections.abc import Callable, Sequence
 from typing import TYPE_CHECKING, Any, cast
 
+from mx_bluesky.common.parameters.components import IspybExperimentType
+from mx_bluesky.common.utils.log import set_dcgid_tag
 from mx_bluesky.hyperion.external_interaction.callbacks.common.ispyb_mapping import (
     populate_data_collection_group,
     populate_remaining_data_collection_info,
@@ -22,8 +24,7 @@ from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_store import (
     IspybIds,
     StoreInIspyb,
 )
-from mx_bluesky.hyperion.log import ISPYB_LOGGER, set_dcgid_tag
-from mx_bluesky.hyperion.parameters.components import IspybExperimentType
+from mx_bluesky.hyperion.log import ISPYB_LOGGER
 from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.parameters.rotation import RotationScan
 
@@ -157,7 +158,7 @@ class RotationISPyBCallback(BaseISPyBCallback):
         data_collection_info = DataCollectionInfo(
             **{
                 f"xtal_snapshot{self._oav_snapshot_event_idx}": data.get(
-                    "oav_snapshot_last_saved_path"
+                    "oav-snapshot-last_saved_path"
                 )
             }
         )

mx_bluesky/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py

@@ -7,8 +7,14 @@ from typing import TYPE_CHECKING, Any
 import numpy as np
 from blueapi.core import MsgGenerator
 from bluesky import preprocessors as bpp
-from dodal.devices.zocalo.zocalo_results import ZOCALO_READING_PLAN_NAME
+from dodal.devices.zocalo.zocalo_results import (
+    ZOCALO_READING_PLAN_NAME,
+    get_processing_results_from_event,
+)
 
+from mx_bluesky.common.parameters.components import DiffractionExperimentWithSample
+from mx_bluesky.common.parameters.constants import PlanNameConstants
+from mx_bluesky.common.utils.log import set_dcgid_tag
 from mx_bluesky.hyperion.external_interaction.callbacks.common.ispyb_mapping import (
     populate_data_collection_group,
     populate_remaining_data_collection_info,
@@ -36,8 +42,7 @@ from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_store import (
     IspybIds,
     StoreInIspyb,
 )
-from mx_bluesky.hyperion.log import ISPYB_LOGGER, set_dcgid_tag
-from mx_bluesky.hyperion.parameters.components import DiffractionExperimentWithSample
+from mx_bluesky.hyperion.log import ISPYB_LOGGER
 from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.parameters.gridscan import (
     GridCommon,
@@ -85,7 +90,7 @@ class GridscanISPyBCallback(BaseISPyBCallback):
         self._processing_start_time: float | None = None
 
     def activity_gated_start(self, doc: RunStart):
-        if doc.get("subplan_name") == CONST.PLAN.DO_FGS:
+        if doc.get("subplan_name") == PlanNameConstants.DO_FGS:
             self._start_of_fgs_uid = doc.get("uid")
         if doc.get("subplan_name") == CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN:
             self.uid_to_finalize_on = doc.get("uid")
@@ -147,7 +152,7 @@ class GridscanISPyBCallback(BaseISPyBCallback):
         ISPYB_LOGGER.info(
             f"Amending comment based on Zocalo reading doc: {format_doc_for_log(doc)}"
         )
-        raw_results = doc["data"]["zocalo-results"]
+        raw_results = get_processing_results_from_event("zocalo", doc)
         if len(raw_results) > 0:
             for n, res in enumerate(raw_results):
                 bb = res["bounding_box"]
@@ -178,25 +183,25 @@ class GridscanISPyBCallback(BaseISPyBCallback):
         data = doc["data"]
         data_collection_id = None
         data_collection_info = DataCollectionInfo(
-            xtal_snapshot1=data.get("oav_grid_snapshot_last_path_full_overlay"),
-            xtal_snapshot2=data.get("oav_grid_snapshot_last_path_outer"),
-            xtal_snapshot3=data.get("oav_grid_snapshot_last_saved_path"),
+            xtal_snapshot1=data.get("oav-grid_snapshot-last_path_full_overlay"),
+            xtal_snapshot2=data.get("oav-grid_snapshot-last_path_outer"),
+            xtal_snapshot3=data.get("oav-grid_snapshot-last_saved_path"),
             n_images=(
-                data["oav_grid_snapshot_num_boxes_x"]
-                * data["oav_grid_snapshot_num_boxes_y"]
+                data["oav-grid_snapshot-num_boxes_x"]
+                * data["oav-grid_snapshot-num_boxes_y"]
             ),
         )
-        microns_per_pixel_x = data["oav_grid_snapshot_microns_per_pixel_x"]
-        microns_per_pixel_y = data["oav_grid_snapshot_microns_per_pixel_y"]
+        microns_per_pixel_x = data["oav-microns_per_pixel_x"]
+        microns_per_pixel_y = data["oav-microns_per_pixel_y"]
         data_collection_grid_info = DataCollectionGridInfo(
-            dx_in_mm=data["oav_grid_snapshot_box_width"] * microns_per_pixel_x / 1000,
-            dy_in_mm=data["oav_grid_snapshot_box_width"] * microns_per_pixel_y / 1000,
-            steps_x=data["oav_grid_snapshot_num_boxes_x"],
-            steps_y=data["oav_grid_snapshot_num_boxes_y"],
+            dx_in_mm=data["oav-grid_snapshot-box_width"] * microns_per_pixel_x / 1000,
+            dy_in_mm=data["oav-grid_snapshot-box_width"] * microns_per_pixel_y / 1000,
+            steps_x=data["oav-grid_snapshot-num_boxes_x"],
+            steps_y=data["oav-grid_snapshot-num_boxes_y"],
             microns_per_pixel_x=microns_per_pixel_x,
             microns_per_pixel_y=microns_per_pixel_y,
-            snapshot_offset_x_pixel=int(data["oav_grid_snapshot_top_left_x"]),
-            snapshot_offset_y_pixel=int(data["oav_grid_snapshot_top_left_y"]),
+            snapshot_offset_x_pixel=int(data["oav-grid_snapshot-top_left_x"]),
+            snapshot_offset_y_pixel=int(data["oav-grid_snapshot-top_left_y"]),
             orientation=Orientation.HORIZONTAL,
             snaked=True,
         )

mx_bluesky/hyperion/external_interaction/config_server.py

@@ -1,5 +1,5 @@
 from daq_config_server.client import ConfigServer
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, model_validator
 
 from mx_bluesky.hyperion.log import LOGGER
 from mx_bluesky.hyperion.parameters.constants import CONST
@@ -17,19 +17,31 @@ def config_server() -> ConfigServer:
 class FeatureFlags(BaseModel):
     # The default value will be used as the fallback when doing a best-effort fetch
     # from the service
-    use_panda_for_gridscan: bool = False
-    use_gpu_for_gridscan: bool = False
-    set_stub_offsets: bool = False
+    use_panda_for_gridscan: bool = CONST.I03.USE_PANDA_FOR_GRIDSCAN
+    compare_cpu_and_gpu_zocalo: bool = CONST.I03.COMPARE_CPU_AND_GPU_ZOCALO
+    set_stub_offsets: bool = CONST.I03.SET_STUB_OFFSETS
 
+    # Feature values supplied at construction will override values from the config server
+    overriden_features: dict = Field(default_factory=dict, exclude=True)
+
+    @model_validator(mode="before")
     @classmethod
-    def _get_flags(cls):
-        flags = config_server().best_effort_get_all_feature_flags()
-        return {f: flags[f] for f in flags if f in cls.__fields__.keys()}
+    def mark_overridden_features(cls, values):
+        assert isinstance(values, dict)
+        values["overriden_features"] = values.copy()
+        return values
 
     @classmethod
-    def best_effort(cls):
-        return cls(**cls._get_flags())
+    def _get_flags(cls):
+        flags = config_server().best_effort_get_all_feature_flags()
+        return {f: flags[f] for f in flags if f in cls.model_fields.keys()}
 
     def update_self_from_server(self):
+        """Used to update the feature flags from the server during a plan. Where there are flags which were explicitly set from externally supplied parameters, these values will be used instead."""
         for flag, value in self._get_flags().items():
-            setattr(self, flag, value)
+            updated_value = (
+                value
+                if flag not in self.overriden_features.keys()
+                else self.overriden_features[flag]
+            )
+            setattr(self, flag, updated_value)

mx_bluesky/hyperion/external_interaction/ispyb/ispyb_store.py

@@ -11,6 +11,7 @@ from ispyb.sp.mxacquisition import MXAcquisition
 from ispyb.strictordereddict import StrictOrderedDict
 from pydantic import BaseModel
 
+from mx_bluesky.common.utils.tracing import TRACER
 from mx_bluesky.hyperion.external_interaction.ispyb.data_model import (
     DataCollectionGridInfo,
     DataCollectionGroupInfo,
@@ -22,7 +23,6 @@ from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_utils import (
     get_session_id_from_visit,
 )
 from mx_bluesky.hyperion.log import ISPYB_LOGGER
-from mx_bluesky.hyperion.tracing import TRACER
 
 if TYPE_CHECKING:
     pass

mx_bluesky/hyperion/external_interaction/ispyb/ispyb_utils.py

@@ -9,8 +9,6 @@ from ispyb.sp.core import Core
 
 from mx_bluesky.hyperion.parameters.constants import CONST
 
-VISIT_PATH_REGEX = r".+/([a-zA-Z]{2}\d{4,5}-\d{1,3})(/?$)"
-
 
 def get_ispyb_config():
     return os.environ.get("ISPYB_CONFIG_PATH", CONST.SIM.ISPYB_CONFIG)

mx_bluesky/hyperion/external_interaction/nexus/nexus_utils.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import time
-from datetime import datetime, timedelta
+from datetime import UTC, datetime, timedelta
 
 import numpy as np
 from dodal.devices.detector import DetectorParams
@@ -93,7 +93,7 @@ def create_goniometer_axes(
 
 def get_start_and_predicted_end_time(time_expected: float) -> tuple[str, str]:
     time_format = r"%Y-%m-%dT%H:%M:%SZ"
-    start = datetime.utcfromtimestamp(time.time())
+    start = datetime.fromtimestamp(time.time(), tz=UTC)
     end_est = start + timedelta(seconds=time_expected)
     return start.strftime(time_format), end_est.strftime(time_format)
 

mx_bluesky/hyperion/external_interaction/nexus/write_nexus.py

@@ -15,12 +15,12 @@ from nexgen.nxs_write.nxmx_writer import NXmxFileWriter
 from numpy.typing import DTypeLike
 from scanspec.core import AxesPoints
 
+from mx_bluesky.common.parameters.components import DiffractionExperimentWithSample
 from mx_bluesky.hyperion.external_interaction.nexus.nexus_utils import (
     create_detector_parameters,
     create_goniometer_axes,
     get_start_and_predicted_end_time,
 )
-from mx_bluesky.hyperion.parameters.components import DiffractionExperimentWithSample
 
 
 class NexusWriter:

mx_bluesky/hyperion/log.py

@@ -1,23 +1,10 @@
 import logging
-from logging.handlers import TimedRotatingFileHandler
-from os import environ
-from pathlib import Path
 
-from dodal.log import (
-    ERROR_LOG_BUFFER_LINES,
-    CircularMemoryHandler,
-    DodalLogHandlers,
-    integrate_bluesky_and_ophyd_logging,
-    set_up_all_logging_handlers,
-)
 from dodal.log import LOGGER as dodal_logger
 
-from mx_bluesky.hyperion.parameters.constants import CONST
-
 LOGGER = logging.getLogger("Hyperion")
 LOGGER.setLevel("DEBUG")
 LOGGER.parent = dodal_logger
-__logger_handlers: DodalLogHandlers | None = None
 
 ISPYB_LOGGER = logging.getLogger("Hyperion ISPyB and Zocalo callbacks")
 ISPYB_LOGGER.setLevel(logging.DEBUG)
@@ -26,74 +13,3 @@ NEXUS_LOGGER = logging.getLogger("Hyperion NeXus callbacks")
 NEXUS_LOGGER.setLevel(logging.DEBUG)
 
 ALL_LOGGERS = [LOGGER, ISPYB_LOGGER, NEXUS_LOGGER]
-
-
-class ExperimentMetadataTagFilter(logging.Filter):
-    dc_group_id: str | None = None
-    run_uid: str | None = None
-
-    def filter(self, record):
-        if self.dc_group_id:
-            record.dc_group_id = self.dc_group_id
-        if self.run_uid:
-            record.run_uid = self.run_uid
-        return True
-
-
-tag_filter = ExperimentMetadataTagFilter()
-
-
-def set_dcgid_tag(dcgid):
-    """Set the datacollection group id as a tag on all subsequent log messages.
-    Setting to None will remove the tag."""
-    tag_filter.dc_group_id = dcgid
-
-
-def set_uid_tag(uid):
-    tag_filter.run_uid = uid
-
-
-def do_default_logging_setup(dev_mode=False):
-    handlers = set_up_all_logging_handlers(
-        dodal_logger,
-        _get_logging_dir(),
-        "hyperion.log",
-        dev_mode,
-        ERROR_LOG_BUFFER_LINES,
-        CONST.GRAYLOG_PORT,
-    )
-    integrate_bluesky_and_ophyd_logging(dodal_logger)
-    handlers["graylog_handler"].addFilter(tag_filter)
-
-    global __logger_handlers
-    __logger_handlers = handlers
-
-
-def _get_debug_handler() -> CircularMemoryHandler:
-    assert (
-        __logger_handlers is not None
-    ), "You can only use this after running the default logging setup"
-    return __logger_handlers["debug_memory_handler"]
-
-
-def flush_debug_handler() -> str:
-    """Writes the contents of the circular debug log buffer to disk and returns the written filename"""
-    handler = _get_debug_handler()
-    assert isinstance(
-        handler.target, TimedRotatingFileHandler
-    ), "Circular memory handler doesn't have an appropriate fileHandler target"
-    handler.flush()
-    return handler.target.baseFilename
-
-
-def _get_logging_dir() -> Path:
-    """Get the path to write the hyperion log files to.
-
-    If the HYPERION_LOG_DIR environment variable exists then logs will be put in here.
-    If no environment variable is found it will default it to the ./tmp/dev directory.
-
-    Returns:
-        logging_path (Path): Path to the log file for the file handler to write to.
-    """
-    logging_path = Path(environ.get("HYPERION_LOG_DIR") or "./tmp/dev/")
-    return logging_path