mx-bluesky 1.2.0__py3-none-any.whl → 1.4.1a0__py3-none-any.whl

mx_bluesky/common/parameters/constants.py

@@ -0,0 +1,138 @@
+from enum import Enum
+
+from dodal.devices.aperturescatterguard import ApertureValue
+from dodal.devices.detector import EIGER2_X_16M_SIZE
+from dodal.devices.zocalo.zocalo_constants import ZOCALO_ENV as ZOCALO_ENV_FROM_DODAL
+from dodal.utils import get_beamline_name
+from pydantic.dataclasses import dataclass
+
+BEAMLINE = get_beamline_name("test")
+TEST_MODE = BEAMLINE == "test"
+
+
+@dataclass(frozen=True)
+class DocDescriptorNames:
+    # Robot load event descriptor
+    ROBOT_LOAD = "robot_load"
+    # For callbacks to use
+    OAV_ROTATION_SNAPSHOT_TRIGGERED = "rotation_snapshot_triggered"
+    OAV_GRID_SNAPSHOT_TRIGGERED = "snapshot_to_ispyb"
+    HARDWARE_READ_PRE = "read_hardware_for_callbacks_pre_collection"
+    HARDWARE_READ_DURING = "read_hardware_for_callbacks_during_collection"
+    ZOCALO_HW_READ = "zocalo_read_hardware_plan"
+    FLYSCAN_RESULTS = "flyscan_results_obtained"
+
+
+@dataclass(frozen=True)
+class OavConstants:
+    OAV_CONFIG_JSON = (
+        "tests/test_data/test_OAVCentring.json"
+        if TEST_MODE
+        else (
+            f"/dls_sw/{BEAMLINE}/software/daq_configuration/json/OAVCentring_hyperion.json"
+        )
+    )
+
+
+@dataclass(frozen=True)
+class PlanNameConstants:
+    # Robot load subplan
+    ROBOT_LOAD = "robot_load"
+    # Gridscan
+    GRID_DETECT_AND_DO_GRIDSCAN = "grid_detect_and_do_gridscan"
+    GRID_DETECT_INNER = "grid_detect"
+    GRIDSCAN_OUTER = "run_gridscan_move_and_tidy"
+    GRIDSCAN_AND_MOVE = "run_gridscan_and_move"
+    GRIDSCAN_MAIN = "run_gridscan"
+    DO_FGS = "do_fgs"
+    # Rotation scan
+    ROTATION_MULTI = "multi_rotation_wrapper"
+    ROTATION_OUTER = "rotation_scan_with_cleanup"
+    ROTATION_MAIN = "rotation_scan_main"
+    FLYSCAN_RESULTS = "xray_centre_results"
+
+
+@dataclass(frozen=True)
+class EnvironmentConstants:
+    ZOCALO_ENV = ZOCALO_ENV_FROM_DODAL
+
+
+@dataclass(frozen=True)
+class TriggerConstants:
+    ZOCALO = "trigger_zocalo_on"
+
+
+@dataclass(frozen=True)
+class HardwareConstants:
+    OAV_REFRESH_DELAY = 0.3
+    PANDA_FGS_RUN_UP_DEFAULT = 0.17
+    CRYOJET_MARGIN_MM = 0.2
+    THAWING_TIME = 20
+
+
+@dataclass(frozen=True)
+class GridscanParamConstants:
+    WIDTH_UM = 600.0
+    EXPOSURE_TIME_S = 0.004
+    USE_ROI = True
+    BOX_WIDTH_UM = 20.0
+    OMEGA_1 = 0.0
+    OMEGA_2 = 90.0
+    PANDA_RUN_UP_DISTANCE_MM = 0.2
+
+
+@dataclass(frozen=True)
+class RotationParamConstants:
+    DEFAULT_APERTURE_POSITION = ApertureValue.LARGE
+
+
+@dataclass(frozen=True)
+class DetectorParamConstants:
+    BEAM_XY_LUT_PATH = (
+        "tests/test_data/test_det_dist_converter.txt"
+        if TEST_MODE
+        else f"/dls_sw/{BEAMLINE}/software/daq_configuration/lookup/DetDistToBeamXYConverter.txt"
+    )
+    DETECTOR = EIGER2_X_16M_SIZE
+
+
+@dataclass(frozen=True)
+class ExperimentParamConstants:
+    DETECTOR = DetectorParamConstants()
+    GRIDSCAN = GridscanParamConstants()
+    ROTATION = RotationParamConstants()
+
+
+@dataclass(frozen=True)
+class PlanGroupCheckpointConstants:
+    # For places to synchronise / stop and wait in plans, use as bluesky group names
+    GRID_READY_FOR_DC = "grid_ready_for_data_collection"
+    ROTATION_READY_FOR_DC = "rotation_ready_for_data_collection"
+    MOVE_GONIO_TO_START = "move_gonio_to_start"
+    READY_FOR_OAV = "ready_for_oav"
+
+
+@dataclass(frozen=True)
+class SimConstants:
+    BEAMLINE = "BL03S"
+    INSERTION_PREFIX = "SR03S"
+    # this one is for unit tests
+    ISPYB_CONFIG = "tests/test_data/test_config.cfg"
+    # this one is for system tests
+    DEV_ISPYB_DATABASE_CFG = "/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-dev.cfg"
+
+
+class Actions(Enum):
+    START = "start"
+    STOP = "stop"
+    SHUTDOWN = "shutdown"
+    STATUS = "status"
+
+
+class Status(Enum):
+    WARN = "Warn"
+    FAILED = "Failed"
+    SUCCESS = "Success"
+    BUSY = "Busy"
+    ABORTING = "Aborting"
+    IDLE = "Idle"

mx_bluesky/common/parameters/gridscan.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import os
+
+from dodal.devices.aperturescatterguard import ApertureValue
+from dodal.devices.detector import (
+    DetectorParams,
+)
+from pydantic import Field
+
+from mx_bluesky.common.parameters.components import (
+    DiffractionExperimentWithSample,
+    IspybExperimentType,
+    OptionalGonioAngleStarts,
+    WithScan,
+    XyzStarts,
+)
+from mx_bluesky.common.parameters.constants import (
+    DetectorParamConstants,
+    GridscanParamConstants,
+    HardwareConstants,
+)
+from mx_bluesky.common.parameters.robot_load import RobotLoadAndEnergyChange
+
+
+class GridCommon(
+    DiffractionExperimentWithSample,
+    OptionalGonioAngleStarts,
+):
+    grid_width_um: float = Field(default=GridscanParamConstants.WIDTH_UM)
+    exposure_time_s: float = Field(default=GridscanParamConstants.EXPOSURE_TIME_S)
+    use_roi_mode: bool = Field(default=GridscanParamConstants.USE_ROI)
+
+    ispyb_experiment_type: IspybExperimentType = Field(
+        default=IspybExperimentType.GRIDSCAN_3D
+    )
+    selected_aperture: ApertureValue | None = Field(default=ApertureValue.SMALL)
+
+    @property
+    def detector_params(self):
+        self.det_dist_to_beam_converter_path = (
+            self.det_dist_to_beam_converter_path
+            or DetectorParamConstants.BEAM_XY_LUT_PATH
+        )
+        optional_args = {}
+        if self.run_number:
+            optional_args["run_number"] = self.run_number
+        assert (
+            self.detector_distance_mm is not None
+        ), "Detector distance must be filled before generating DetectorParams"
+        os.makedirs(self.storage_directory, exist_ok=True)
+        return DetectorParams(
+            detector_size_constants=DetectorParamConstants.DETECTOR,
+            expected_energy_ev=self.demand_energy_ev,
+            exposure_time=self.exposure_time_s,
+            directory=self.storage_directory,
+            prefix=self.file_name,
+            detector_distance=self.detector_distance_mm,
+            omega_start=self.omega_start_deg or 0,
+            omega_increment=0,
+            num_images_per_trigger=1,
+            num_triggers=self.num_images,
+            use_roi_mode=self.use_roi_mode,
+            det_dist_to_beam_converter_path=self.det_dist_to_beam_converter_path,
+            trigger_mode=self.trigger_mode,
+            **optional_args,
+        )
+
+
+class RobotLoadThenCentre(GridCommon):
+    thawing_time: float = Field(default=HardwareConstants.THAWING_TIME)
+
+    def robot_load_params(self):
+        my_params = self.model_dump()
+        return RobotLoadAndEnergyChange(**my_params)
+
+    def pin_centre_then_xray_centre_params(self):
+        my_params = self.model_dump()
+        del my_params["thawing_time"]
+        return PinTipCentreThenXrayCentre(**my_params)
+
+
+class GridScanWithEdgeDetect(GridCommon):
+    box_size_um: float = Field(default=GridscanParamConstants.BOX_WIDTH_UM)
+
+
+class PinTipCentreThenXrayCentre(GridCommon):
+    tip_offset_um: float = 0
+
+
+class SpecifiedGrid(XyzStarts, WithScan):
+    """A specified grid is one which has defined values for the start position,
+    grid and box sizes, etc., as opposed to parameters for a plan which will create
+    those parameters at some point (e.g. through optical pin detection)."""

mx_bluesky/common/parameters/robot_load.py

@@ -0,0 +1,16 @@
+from pydantic import Field
+
+from mx_bluesky.common.parameters.components import (
+    MxBlueskyParameters,
+    WithOptionalEnergyChange,
+    WithSample,
+    WithSnapshot,
+    WithVisit,
+)
+from mx_bluesky.common.parameters.constants import HardwareConstants
+
+
+class RobotLoadAndEnergyChange(
+    MxBlueskyParameters, WithSample, WithSnapshot, WithOptionalEnergyChange, WithVisit
+):
+    thawing_time: float = Field(default=HardwareConstants.THAWING_TIME)

mx_bluesky/common/plans/__init__.py

@@ -0,0 +1 @@
+"""Common MX plans which includes open and close data collection runs. These be used in isolation or as part of a larger plan"""

mx_bluesky/common/plans/do_fgs.py

@@ -0,0 +1,121 @@
+from collections.abc import Callable
+from time import time
+
+import bluesky.plan_stubs as bps
+import bluesky.preprocessors as bpp
+from bluesky.utils import MsgGenerator
+from dodal.devices.eiger import EigerDetector
+from dodal.devices.fast_grid_scan import FastGridScanCommon
+from dodal.devices.synchrotron import Synchrotron
+from dodal.devices.zocalo.zocalo_results import (
+    ZOCALO_STAGE_GROUP,
+)
+from dodal.log import LOGGER
+from dodal.plan_stubs.check_topup import check_topup_and_wait_if_necessary
+from scanspec.core import AxesPoints, Axis
+
+from mx_bluesky.common.device_setup_plans.read_hardware_for_setup import (
+    read_hardware_for_zocalo,
+)
+from mx_bluesky.common.parameters.constants import (
+    EnvironmentConstants,
+    PlanNameConstants,
+    TriggerConstants,
+)
+from mx_bluesky.common.utils.tracing import TRACER
+
+
+def _wait_for_zocalo_to_stage_then_do_fgs(
+    grid_scan_device: FastGridScanCommon,
+    detector: EigerDetector,
+    synchrotron: Synchrotron,
+    during_collection_plan: Callable[[], MsgGenerator] | None = None,
+):
+    expected_images = yield from bps.rd(grid_scan_device.expected_images)
+    exposure_sec_per_image = yield from bps.rd(detector.cam.acquire_time)  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
+    LOGGER.info("waiting for topup if necessary...")
+    yield from check_topup_and_wait_if_necessary(
+        synchrotron,
+        expected_images * exposure_sec_per_image,
+        30.0,
+    )
+
+    # Make sure ZocaloResults queue is clear and ready to accept our new data. Zocalo MUST
+    # have been staged using ZOCALO_STAGE_GROUP prior to this
+    LOGGER.info("Waiting for Zocalo device queue to have been cleared...")
+    yield from bps.wait(ZOCALO_STAGE_GROUP)
+
+    # Triggers Zocalo if RE is subscribed to ZocaloCallback
+    yield from read_hardware_for_zocalo(detector)
+    LOGGER.info("Wait for all moves with no assigned group")
+    yield from bps.wait()
+
+    LOGGER.info("kicking off FGS")
+    yield from bps.kickoff(grid_scan_device, wait=True)
+    gridscan_start_time = time()
+    if during_collection_plan:
+        yield from during_collection_plan()
+    LOGGER.info("completing FGS")
+    yield from bps.complete(grid_scan_device, wait=True)
+    # Remove this logging statement once metrics have been added
+    LOGGER.info(
+        f"Grid scan motion program took {round(time()-gridscan_start_time,2)} to complete"
+    )
+
+
+def kickoff_and_complete_gridscan(
+    gridscan: FastGridScanCommon,
+    detector: EigerDetector,  # Once Eiger inherits from StandardDetector, use that type instead
+    synchrotron: Synchrotron,
+    scan_points: list[AxesPoints[Axis]],
+    scan_start_indices: list[int],
+    plan_during_collection: Callable[[], MsgGenerator] | None = None,
+    zocalo_environment: str = EnvironmentConstants.ZOCALO_ENV,
+):
+    """Triggers a grid scan motion program and waits for completion, accounting for synchrotron topup.
+    If the RunEngine is subscribed to ZocaloCallback, this plan will also trigger Zocalo.
+
+    Can be used for multiple successive grid scans, see Hyperion's usage
+
+    Args:
+        gridscan (FastGridScanCommon):          Device which can trigger a fast grid scan and wait for completion
+        detector (EigerDetector)                Detector device
+        synchrotron (Synchrotron):              Synchrotron device
+        scan_points (list[AxesPoints[Axis]]):   Each element in the list contains all the grid points for that grid scan.
+                                                Two elements in this list indicates that two grid scans will be done, eg for Hyperion's 3D grid scans.
+        scan_start_indices (list[int]):         Contains the first index of each grid scan
+        plan_during_collection (Optional, MsgGenerator): Generic plan called in between kickoff and completion,
+                                                eg waiting on zocalo.
+        zocalo_environment (Optional, str)      Used for zocalo connection
+    """
+
+    assert len(scan_points) == len(
+        scan_start_indices
+    ), "scan_points and scan_start_indices must be lists of the same length!"
+
+    plan_name = PlanNameConstants.DO_FGS
+
+    @TRACER.start_as_current_span(plan_name)
+    @bpp.set_run_key_decorator(plan_name)
+    @bpp.run_decorator(
+        md={
+            "subplan_name": plan_name,
+            TriggerConstants.ZOCALO: plan_name,
+            "scan_points": scan_points,
+            "scan_start_indices": scan_start_indices,
+            "zocalo_environment": zocalo_environment,
+        }
+    )
+    @bpp.contingency_decorator(
+        except_plan=lambda e: (yield from bps.stop(detector)),  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
+        else_plan=lambda: (yield from bps.unstage(detector)),  # type: ignore # See: https://github.com/bluesky/bluesky/issues/1809
+    )
+    def _decorated_do_fgs():
+        yield from _wait_for_zocalo_to_stage_then_do_fgs(
+            gridscan,
+            detector,
+            synchrotron,
+            during_collection_plan=plan_during_collection,
+        )
+
+    yield from _decorated_do_fgs()

mx_bluesky/common/utils/log.py

@@ -0,0 +1,118 @@
+import logging
+from logging.handlers import TimedRotatingFileHandler
+from os import environ
+from pathlib import Path
+
+from dodal.log import (
+    ERROR_LOG_BUFFER_LINES,
+    CircularMemoryHandler,
+    DodalLogHandlers,
+    integrate_bluesky_and_ophyd_logging,
+    set_up_all_logging_handlers,
+)
+from dodal.log import LOGGER as dodal_logger
+
+LOGGER = logging.getLogger("mx-bluesky")
+
+__logger_handlers: DodalLogHandlers | None = None
+
+
+class ExperimentMetadataTagFilter(logging.Filter):
+    """When an instance of this custom filter is added to a logging handler, dc_group_id
+    and run_id will be tagged in that handlers' log messages."""
+
+    dc_group_id: str | None = None
+    run_uid: str | None = None
+
+    def filter(self, record):
+        if self.dc_group_id:
+            record.dc_group_id = self.dc_group_id
+        if self.run_uid:
+            record.run_uid = self.run_uid
+        return True
+
+
+tag_filter = ExperimentMetadataTagFilter()
+
+
+def set_dcgid_tag(dcgid):
+    """Set the datacollection group id as a tag on all subsequent log messages.
+    Setting to None will remove the tag."""
+    tag_filter.dc_group_id = dcgid
+
+
+def set_uid_tag(uid):
+    """Set the unique id as a tag on all subsequent log messages.
+    Setting to None will remove the tag."""
+    tag_filter.run_uid = uid
+
+
+def do_default_logging_setup(
+    file_name: str,
+    graylog_port: int,
+    dev_mode: bool = False,
+    integrate_all_logs: bool = True,
+):
+    """Configures dodal logger so that separate debug and info log files are created,
+    info logs are sent to Graylog, info logs are streamed to sys.sterr, and logs from ophyd
+    and bluesky and ophyd-async are optionally included."""
+
+    handlers = set_up_all_logging_handlers(
+        dodal_logger,
+        _get_logging_dir(),
+        file_name,
+        dev_mode,
+        ERROR_LOG_BUFFER_LINES,
+        graylog_port,
+    )
+
+    if integrate_all_logs:
+        integrate_bluesky_and_ophyd_logging(dodal_logger)
+
+    handlers["graylog_handler"].addFilter(tag_filter)
+
+    global __logger_handlers
+    __logger_handlers = handlers
+
+
+def _get_debug_handler() -> CircularMemoryHandler:
+    assert (
+        __logger_handlers is not None
+    ), "You can only use this after running the default logging setup"
+    return __logger_handlers["debug_memory_handler"]
+
+
+def flush_debug_handler() -> str:
+    """Writes the contents of the circular debug log buffer to disk and returns the written filename"""
+    handler = _get_debug_handler()
+    assert isinstance(
+        handler.target, TimedRotatingFileHandler
+    ), "Circular memory handler doesn't have an appropriate fileHandler target"
+    handler.flush()
+    return handler.target.baseFilename
+
+
+def _get_logging_dir() -> Path:
+    """Get the path to write the mx_bluesky log files to.
+
+    Log location can be specified in the LOG_DIR environment variable, otherwise MX bluesky logs are written to 'dls_sw/ixx/logs/bluesky'.
+    This directory will be created if it is not found
+
+    Logs are written to ./tmp/logs/bluesky if BEAMLINE environment variable is not found
+
+    Returns:
+        logging_path (Path): Path to the log file for the file handler to write to.
+    """
+
+    logging_str = environ.get("LOG_DIR")
+    if logging_str:
+        logging_path = Path(logging_str)
+    else:
+        beamline = environ.get("BEAMLINE")
+        logging_path = (
+            Path(f"/dls_sw/{beamline}/logs/bluesky/")
+            if beamline
+            else Path("/tmp/logs/bluesky")
+        )
+    Path.mkdir(logging_path, exist_ok=True, parents=True)
+    return logging_path

mx_bluesky/{hyperion → common/utils}/tracing.py

@@ -8,8 +8,8 @@ from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
 
-def setup_tracing():
-    resource = Resource(attributes={SERVICE_NAME: "Hyperion"})
+def setup_tracing(service_name: str = "Hyperion"):
+    resource = Resource(attributes={SERVICE_NAME: service_name})
 
     traceProvider = TracerProvider(resource=resource)
     processor = BatchSpanProcessor(

mx_bluesky/hyperion/__main__.py

@@ -7,13 +7,18 @@ from queue import Queue
 from traceback import format_exception
 from typing import Any
 
-from blueapi.core import BlueskyContext, MsgGenerator
+from blueapi.core import BlueskyContext
 from bluesky.callbacks.zmq import Publisher
 from bluesky.run_engine import RunEngine
+from bluesky.utils import MsgGenerator
 from flask import Flask, request
 from flask_restful import Api, Resource
 from pydantic.dataclasses import dataclass
 
+from mx_bluesky.common.parameters.components import MxBlueskyParameters
+from mx_bluesky.common.parameters.constants import Actions, Status
+from mx_bluesky.common.utils.log import do_default_logging_setup, flush_debug_handler
+from mx_bluesky.common.utils.tracing import TRACER
 from mx_bluesky.hyperion.exceptions import WarningException
 from mx_bluesky.hyperion.experiment_plans.experiment_registry import (
     PLAN_REGISTRY,
@@ -36,13 +41,9 @@ from mx_bluesky.hyperion.external_interaction.callbacks.logging_callback import
 )
 from mx_bluesky.hyperion.log import (
     LOGGER,
-    do_default_logging_setup,
-    flush_debug_handler,
 )
 from mx_bluesky.hyperion.parameters.cli import parse_cli_args
-from mx_bluesky.hyperion.parameters.components import HyperionParameters
-from mx_bluesky.hyperion.parameters.constants import CONST, Actions, Status
-from mx_bluesky.hyperion.tracing import TRACER
+from mx_bluesky.hyperion.parameters.constants import CONST
 from mx_bluesky.hyperion.utils.context import setup_context
 
 VERBOSE_EVENT_LOGGING: bool | None = None
@@ -53,7 +54,7 @@ class Command:
     action: Actions
     devices: Any | None = None
     experiment: Callable[[Any, Any], MsgGenerator] | None = None
-    parameters: HyperionParameters | None = None
+    parameters: MxBlueskyParameters | None = None
     callbacks: CallbacksFactory | None = None
 
 
@@ -119,7 +120,7 @@ class BlueskyRunner:
     def start(
         self,
         experiment: Callable,
-        parameters: HyperionParameters,
+        parameters: MxBlueskyParameters,
         plan_name: str,
         callbacks: CallbacksFactory | None,
     ) -> StatusAndMessage:
@@ -239,7 +240,7 @@ def compose_start_args(context: BlueskyContext, plan_name: str, action: Actions)
             raise ValueError(f"Extra fields not allowed {parameters.model_extra}")
     except Exception as e:
         raise ValueError(
-            f"Supplied parameters don't match the plan for this endpoint {request.data}"
+            f"Supplied parameters don't match the plan for this endpoint {request.data}, for plan {plan_name}"
         ) from e
     return plan, parameters, plan_name, callback_type
 
@@ -345,7 +346,9 @@ def create_app(
 def create_targets():
     hyperion_port = 5005
     args = parse_cli_args()
-    do_default_logging_setup(dev_mode=args.dev_mode)
+    do_default_logging_setup(
+        CONST.LOG_FILE_NAME, CONST.GRAYLOG_PORT, dev_mode=args.dev_mode
+    )
     if not args.use_external_callbacks:
         setup_callback_logging(args.dev_mode)
     app, runner = create_app(