mx-bluesky 1.4.1a0__tar.gz → 1.4.3__tar.gz

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/.github/workflows/_container.yml

@@ -51,6 +51,7 @@ jobs:
         # Docker cache and publish it
         with:
           context: .
+          file: Dockerfile.release
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/.gitignore

@@ -172,3 +172,6 @@ edm_serial/
 **/parameters/fixed_target/*/*.map
 # idea project files
 .idea/
+
+# Generated PlantUML diagrams
+docs/developer/hyperion/reference/param_hierarchy.puml

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mx-bluesky
-Version: 1.4.1a0
+Version: 1.4.3
 Summary: Bluesky tools for MX Beamlines at DLS
 Author-email: Dominic Oram <dominic.oram@diamond.ac.uk>
 License:                                  Apache License
@@ -226,7 +226,7 @@ Requires-Dist: opencv-python
 Requires-Dist: opentelemetry-distro
 Requires-Dist: opentelemetry-exporter-otlp
 Requires-Dist: pydantic
-Requires-Dist: pydantic-extra-types
+Requires-Dist: pydantic-extra-types>=2.10.1
 Requires-Dist: pyepics
 Requires-Dist: pyzmq
 Requires-Dist: requests
@@ -238,8 +238,8 @@ Requires-Dist: blueapi>=0.5.0
 Requires-Dist: daq-config-server>=0.1.1
 Requires-Dist: ophyd==1.9.0
 Requires-Dist: ophyd-async>=0.8a5
-Requires-Dist: bluesky>=1.13.0a4
-Requires-Dist: dls-dodal==1.36.1a
+Requires-Dist: bluesky>=1.13
+Requires-Dist: dls-dodal==1.38.0
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: build; extra == "dev"
@@ -250,6 +250,7 @@ Requires-Dist: ipython; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Requires-Dist: myst-parser; extra == "dev"
 Requires-Dist: pipdeptree; extra == "dev"
+Requires-Dist: plantweb; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: pydata-sphinx-theme>=0.12; extra == "dev"
 Requires-Dist: pyright; extra == "dev"
@@ -260,7 +261,6 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: sphinx-autobuild; extra == "dev"
 Requires-Dist: sphinx-copybutton; extra == "dev"
-Requires-Dist: sphinxcontrib-plantuml; extra == "dev"
 Requires-Dist: sphinx-design; extra == "dev"
 Requires-Dist: tox-direct; extra == "dev"
 Requires-Dist: tox; extra == "dev"

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/docs/conf.py

@@ -49,6 +49,8 @@ extensions = [
     "sphinx_design",
     # For markdown
     "myst_parser",
+    # For plantUML diagrams
+    "plantweb.directive",
 ]
 
 myst_enable_extensions = [

mx_bluesky-1.4.3/docs/developer/hyperion/reference/gridscan.puml

@@ -0,0 +1,109 @@
+@startuml
+title Gridscan Parameter Relationships
+
+class DiffractionExperiment
+class DiffractionExperimentWithSample
+class GridCommon {
+    grid_width_um
+    exposure_time_s
+}
+class GridScanWithEdgeDetect {
+    box_size_um
+}
+class HyperionGridCommon {
+    enable_dev_shm
+}
+class HyperionThreeDGridScan {
+    x_step_size_um
+    y_step_size_um
+    z_step_size_um
+    y2_start_um
+    z2_start_um
+    --
+    grid_1_spec
+    grid_2_spec
+    scan_indices
+    scan_spec
+    scan_points
+    scan_points_first_grid
+    scan_points_second_grid
+    num_images
+    FGS_Params
+    panda_FGS_Params
+}
+class MxBlueSkyParameters
+class SpecifiedGrid
+class XyzStarts {
+    x_start_um
+    y_start_um
+    z_start_um
+}
+class OptionalXYZStarts {
+    x_start_um
+    y_start_um
+    z_start_um
+}
+class RotationScanPerSweep
+
+MxBlueSkyParameters <|-- DiffractionExperiment
+DiffractionExperiment <|-- DiffractionExperimentWithSample
+DiffractionExperimentWithSample <|-- GridCommon
+GridCommon <|-- GridScanWithEdgeDetect
+GridCommon <|-- HyperionGridCommon
+HyperionGridCommon <|-- HyperionThreeDGridScan
+SpecifiedGrid <|-- HyperionThreeDGridScan
+XyzStarts <|-- SpecifiedGrid
+OptionalXYZStarts <|-- RotationScanPerSweep
+class GridParamUpdate {
+    x_start_um
+    y_start_um
+    y2_start_um
+    z_start_um
+    z2_start_um
+    x_steps
+    y_steps
+    z_steps
+    x_step_size_um
+    y_step_size_um
+    z_step_size_um
+}
+
+class GridDetectionCallback {
+    get_grid_parameters() -> GridParamUpdate
+}
+    
+GridDetectionCallback --> GridParamUpdate : generates from event. Adds 0.5 to get box-centres
+GridParamUpdate --> HyperionThreeDGridScan : combines with GridScanWithEdgeDetect
+
+class experiment_plans {
+    grid_detect_then_xray_centre()
+    flyscan_xray_centre_no_move()
+    create_parameters_for_flyscan_xray_centre(GridScanWithEdgeDetect, GridParamUpdate) -> HyperionThreeDGridScan
+}
+
+class AbstractExperimentBase
+class AbstractExperimentWithBeamParams
+class GridScanParamsCommon {
+    x_steps
+    y_steps
+    z_steps
+    x_step_size_mm
+    y_step_size_mm
+    z_step_size_mm
+    x_start_mm
+    y1_start_mm
+    y2_start_mm
+    z1_start_mm
+    z2_start_mm
+}
+class PandAGridScanParams
+class ZebraGridScanParams
+
+AbstractExperimentBase <|-- AbstractExperimentWithBeamParams
+AbstractExperimentWithBeamParams <|-- GridScanParamsCommon
+GridScanParamsCommon <|-- PandAGridScanParams
+GridScanParamsCommon <|-- ZebraGridScanParams
+
+HyperionThreeDGridScan --> ZebraGridScanParams : generates
+HyperionThreeDGridScan --> PandAGridScanParams : generates
+@enduml

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/docs/developer/hyperion/reference/param-hierarchy.rst

@@ -1,4 +1,4 @@
 Hyperion Parameter class hierarchy
 ==================================
 
-TODO: automate including the param hierarchy here
+.. uml:: param_hierarchy.puml

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/pyproject.toml

@@ -25,11 +25,11 @@ dependencies = [
     "matplotlib",
     "nexgen",
     "numpy",
-    "opencv-python",               # Needed for I24 ssx moveonclick. To be changed to headless once this is moved to separate ui.
+    "opencv-python",                  # Needed for I24 ssx moveonclick. To be changed to headless once this is moved to separate ui.
     "opentelemetry-distro",
     "opentelemetry-exporter-otlp",
     "pydantic",
-    "pydantic-extra-types",
+    "pydantic-extra-types >= 2.10.1",
     "pyepics",
     "pyzmq",
     "requests",
@@ -45,8 +45,8 @@ dependencies = [
     "daq-config-server >= 0.1.1",
     "ophyd == 1.9.0",
     "ophyd-async >= 0.8a5",
-    "bluesky >= 1.13.0a4",
-    "dls-dodal == 1.36.1a",
+    "bluesky >= 1.13",
+    "dls-dodal == 1.38.0",
 ]
 
 
@@ -68,6 +68,7 @@ dev = [
     "mypy",
     "myst-parser",
     "pipdeptree",
+    "plantweb",
     "pre-commit",
     "pydata-sphinx-theme>=0.12",
     "pyright",
@@ -78,7 +79,6 @@ dev = [
     "ruff",
     "sphinx-autobuild",
     "sphinx-copybutton",
-    "sphinxcontrib-plantuml",
     "sphinx-design",
     "tox-direct",
     "tox",
@@ -180,6 +180,9 @@ commands =
     type-checking: pyright src tests {posargs}
     tests: pytest --cov=mx_bluesky --cov-report term --cov-report xml:cov.xml {posargs}
     docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html
+commands_pre =
+    docs: /usr/bin/bash -c "{toxinidir}/utility_scripts/generate_plantuml.py > \
+        docs/developer/hyperion/reference/param_hierarchy.puml"
 """
 
 [tool.ruff]

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/src/mx_bluesky/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.1a0'
-__version_tuple__ = version_tuple = (1, 4, 1)
+__version__ = version = '1.4.3'
+__version_tuple__ = version_tuple = (1, 4, 3)

mx_bluesky-1.4.3/src/mx_bluesky/beamlines/i04/redis_to_murko_forwarder.py

@@ -0,0 +1,178 @@
+import io
+import json
+import pickle
+from typing import TypedDict
+
+import numpy as np
+import zmq
+from dodal.beamlines.i04 import MURKO_REDIS_DB, REDIS_HOST, REDIS_PASSWORD
+from numpy.typing import NDArray
+from PIL import Image
+from redis import StrictRedis
+
+from mx_bluesky.common.utils.log import LOGGER
+
+MURKO_ADDRESS = "tcp://i04-murko-prod.diamond.ac.uk:8008"
+
+
+class MurkoRequest(TypedDict):
+    to_predict: NDArray
+    model_img_size: tuple[int, int]
+    save: bool
+    min_size: int
+    description: list
+    prefix: list[str]
+
+
+def get_image_size(image: NDArray) -> tuple[int, int]:
+    """Returns the width and height of a numpy image"""
+    return image.shape[1], image.shape[0]
+
+
+def send_to_murko_and_get_results(request: MurkoRequest) -> dict:
+    LOGGER.info(f"Sending {request['prefix']} to murko")
+    context = zmq.Context()
+    socket = context.socket(zmq.REQ)
+    socket.connect(MURKO_ADDRESS)
+    socket.send(pickle.dumps(request))
+    raw_results = socket.recv()
+    results = pickle.loads(raw_results)
+    LOGGER.info(f"Got {len(results['descriptions'])} results")
+    return results
+
+
+def correlate_results_to_uuids(request: MurkoRequest, murko_results: dict) -> list:
+    results = []
+    uuids = request["prefix"]
+
+    width, height = get_image_size(request["to_predict"][0])
+
+    for uuid, prediction in zip(uuids, murko_results["descriptions"], strict=False):
+        coords = prediction["most_likely_click"]
+        y_coord = coords[0] * height
+        x_coord = coords[1] * width
+        results.append(
+            {"uuid": uuid, "x_pixel_coord": x_coord, "y_pixel_coord": y_coord}
+        )
+    return results
+
+
+class BatchMurkoForwarder:
+    def __init__(self, redis_client: StrictRedis, batch_size: int):
+        """
+        Holds image data streamed from redis and forwards it to murko when:
+            * A set number have been received
+            * The shape of the images changes
+            * When `flush` is called
+
+        Once data has been forwarded this will then wait on the results and put them
+        back in redis.
+
+        Args:
+            redis_client: The client to send murko results back to redis.
+            batch_size: How many results to accumulate until they are flushed to redis.
+        """
+        self.redis_client = redis_client
+        self.batch_size = batch_size
+        self._uuids_and_images: dict[str, NDArray] = {}
+        self._last_image_size: tuple[int, int] | None = None
+        self._last_sample_id = ""
+
+    def _handle_batch_of_images(self, sample_id, images, uuids):
+        request_arguments: MurkoRequest = {
+            "model_img_size": (256, 320),
+            "to_predict": np.array(images),
+            "save": False,
+            "min_size": 64,
+            "description": [
+                "foreground",
+                "crystal",
+                "loop_inside",
+                "loop",
+                ["crystal", "loop"],
+                ["crystal", "loop", "stem"],
+            ],
+            "prefix": uuids,
+        }
+        predictions = send_to_murko_and_get_results(request_arguments)
+        results = correlate_results_to_uuids(request_arguments, predictions)
+        self._send_murko_results_to_redis(sample_id, results)
+
+    def _send_murko_results_to_redis(self, sample_id: str, results: list):
+        for result in results:
+            self.redis_client.hset(
+                f"murko:{sample_id}:results", result["uuid"], json.dumps(result)
+            )
+        self.redis_client.publish("murko-results", json.dumps(results))
+
+    def add(self, sample_id: str, uuid: str, image: NDArray):
+        """Add an image to the batch to send to murko."""
+        image_size = get_image_size(image)
+        self._last_sample_id = sample_id
+        if self._last_image_size and self._last_image_size != image_size:
+            self.flush()
+        self._uuids_and_images[uuid] = image
+        self._last_image_size = image_size
+        if len(self._uuids_and_images.keys()) >= self.batch_size:
+            self.flush()
+
+    def flush(self):
+        """Flush the batch to murko."""
+        if self._uuids_and_images:
+            self._handle_batch_of_images(
+                self._last_sample_id,
+                list(self._uuids_and_images.values()),
+                list(self._uuids_and_images.keys()),
+            )
+        self._uuids_and_images = {}
+        self._last_image_size = None
+
+
+class RedisListener:
+    TIMEOUT_S = 2
+
+    def __init__(
+        self,
+        redis_host=REDIS_HOST,
+        redis_password=REDIS_PASSWORD,
+        db=MURKO_REDIS_DB,
+        redis_channel="murko",
+    ):
+        self.redis_client = StrictRedis(
+            host=redis_host,
+            password=redis_password,
+            db=db,
+        )
+        self.pubsub = self.redis_client.pubsub()
+        self.channel = redis_channel
+        self.forwarder = BatchMurkoForwarder(self.redis_client, 10)
+
+    def _get_and_handle_message(self):
+        message = self.pubsub.get_message(timeout=self.TIMEOUT_S)
+        if message and message["type"] == "message":
+            data = json.loads(message["data"])
+            LOGGER.info(f"Received from redis: {data}")
+            uuid = data["uuid"]
+            sample_id = data["sample_id"]
+
+            # Images are put in redis as raw jpeg bytes, murko needs numpy arrays
+            raw_image = self.redis_client.hget(f"murko:{sample_id}:raw", uuid)
+            assert isinstance(raw_image, bytes)
+            image = Image.open(io.BytesIO(raw_image))
+            image = np.asarray(image)
+
+            self.forwarder.add(sample_id, uuid, image)
+
+        elif not message:
+            self.forwarder.flush()
+
+    def listen_for_image_data_forever(self):
+        self.pubsub.subscribe(self.channel)
+
+        while True:
+            self._get_and_handle_message()
+
+
+if __name__ == "__main__":
+    client = RedisListener()
+    client.listen_for_image_data_forever()

{mx_bluesky-1.4.1a0 → mx_bluesky-1.4.3}/src/mx_bluesky/beamlines/i24/serial/__init__.py

@@ -18,9 +18,6 @@ from .fixed_target.i24ssx_Chip_Manager_py3v1 import (
     moveto,
     moveto_preset,
     pumpprobe_calc,
-    save_screen_map,
-    upload_parameters,
-    write_parameter_file,
 )
 from .log import clean_up_log_config_at_end, setup_collection_logs
 from .setup_beamline.setup_detector import setup_detector_stage
@@ -44,9 +41,6 @@ __all__ = [
     "load_lite_map",
     "load_stock_map",
     "pumpprobe_calc",
-    "save_screen_map",
-    "upload_parameters",
-    "write_parameter_file",
     "setup_collection_logs",
     "clean_up_log_config_at_end",
 ]