mx-bluesky 1.4.0__tar.gz → 1.4.1__tar.gz

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/.github/workflows/_container.yml

@@ -30,12 +30,8 @@ jobs:
           DOCKER_BUILD_RECORD_UPLOAD: false
         with:
           context: .
-          # Need load and tags so we can test it below
+          file: Dockerfile.release
           load: true
-          tags: tag_for_testing
-
-      - name: Test cli works in cached runtime image
-        run: docker run --rm tag_for_testing --version
 
       - name: Create tags for publishing image
         id: meta

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/Dockerfile.release

@@ -33,6 +33,6 @@ RUN git restore .
 RUN rm src/mx_bluesky/_version.py
 RUN pip install --no-cache-dir --no-compile --no-deps -e .
 
-ENTRYPOINT /app/hyperion/utility_scripts/docker/entrypoint.sh
+ENTRYPOINT ["/app/hyperion/utility_scripts/docker/entrypoint.sh"]
 
 EXPOSE 5005

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mx-bluesky
-Version: 1.4.0
+Version: 1.4.1
 Summary: Bluesky tools for MX Beamlines at DLS
 Author-email: Dominic Oram <dominic.oram@diamond.ac.uk>
 License:                                  Apache License
@@ -237,9 +237,9 @@ Requires-Dist: matplotlib
 Requires-Dist: blueapi>=0.5.0
 Requires-Dist: daq-config-server>=0.1.1
 Requires-Dist: ophyd==1.9.0
-Requires-Dist: ophyd-async>=0.3a5
+Requires-Dist: ophyd-async>=0.8a5
 Requires-Dist: bluesky>=1.13.0a4
-Requires-Dist: dls-dodal==1.35.0
+Requires-Dist: dls-dodal==1.36.2
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: build; extra == "dev"

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/docs/conf.py

@@ -136,6 +136,7 @@ rst_epilog = """
 .. _black: https://github.com/psf/black
 .. _mypy: http://mypy-lang.org/
 .. _pre-commit: https://pre-commit.com/
+.. _ruff: https://github.com/astral-sh/ruff
 """
 
 # Ignore localhost links for periodic check that links in docs are valid

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/docs/developer/general/explanations/decisions/0003-python-version-support.rst

@@ -1,4 +1,4 @@
-1. Python version support
+3. Python version support
 =========================
 
 Date: 2024-08-22

mx_bluesky-1.4.1/docs/developer/general/how-to/dev-ops/dev-ops.rst

@@ -0,0 +1,8 @@
+Dev-Ops guides
+==============================
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    ./*

{mx_bluesky-1.4.0/docs/developer/general/how-to → mx_bluesky-1.4.1/docs/developer/general/how-to/dev-ops}/lint.rst

@@ -1,7 +1,7 @@
 Run linting using pre-commit
 ============================
 
-Code linting is handled by black_, ruff run under pre-commit_.
+Code linting is handled by ruff_, run under pre-commit_.
 
 Running pre-commit
 ------------------
@@ -18,15 +18,15 @@ commit`` on just the files that have changed::
 Fixing issues
 -------------
 
-If black reports an issue you can tell it to reformat all the files in the
+If ruff reports an issue you can attempt to automatically reformat all the files in the
 repository::
 
-    $ black .
+    $ ruff check --fix .
 
 If you get any ruff issues you will have to fix those manually.
 
 VSCode support
 --------------
 
-The ``.vscode/settings.json`` will run black and isort formatters as well as
-flake8 checking on save. Issues will be highlighted in the editor window.
+The ``.vscode/settings.json`` will run linters and formatters 
+on save. Issues will be highlighted in the editor window.

{mx_bluesky-1.4.0/docs/developer/general/how-to → mx_bluesky-1.4.1/docs/developer/general/how-to/dev-ops}/run-tests.rst

@@ -4,7 +4,7 @@ Run the tests using pytest
 Testing is done with pytest_. It will find functions in the project that `look
 like tests`_, and run them to check for errors. You can run it with::
 
-    $ tox -e pytest
+    $ tox -e tests
 
 It will also report coverage to the commandline and to ``cov.xml``.
 

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/docs/developer/general/index.rst

@@ -12,7 +12,7 @@ Documentation is split into four categories, and each is also accessible from li
             :caption: Tutorials
             :maxdepth: 1
 
-            tutorials/dev-install
+            tutorials/profile-tests
 
         +++
 
@@ -26,14 +26,9 @@ Documentation is split into four categories, and each is also accessible from li
 
             how-to/get-started
             how-to/contribute
-            how-to/build-docs
-            how-to/run-tests
-            how-to/static-analysis
-            how-to/lint
-            how-to/update-tools
             how-to/create-a-release
             how-to/deploy-a-release
-            how-to/profile-tests
+            how-to/dev-ops/dev-ops
 
         +++
 

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/docs/developer/general/reference/standards.rst

@@ -15,7 +15,7 @@ The code in this repository conforms to standards set by the following tools:
 
 .. seealso::
 
-    How-to guides `../how-to/lint` and `../how-to/static-analysis`
+    How-to guides `../how-to/dev-ops/lint`
 
 .. _documentation_standards:
 
@@ -60,4 +60,4 @@ Docs follow the underlining convention::
 
 .. seealso::
 
-    How-to guide `../how-to/build-docs`
+    How-to guide `../how-to/dev-ops/build-docs`

{mx_bluesky-1.4.0/docs/developer/general/how-to → mx_bluesky-1.4.1/docs/developer/general/tutorials}/profile-tests.rst

@@ -15,7 +15,7 @@ value e.g.
 0.1s
 
 How to profile the unit tests
-=============================
+-----------------------------
 
 Sometimes tests can exhibit slowdowns for unknown reasons, in order to diagnose the cause it can be useful to profile
 the test code.
@@ -35,7 +35,7 @@ You can then often step through in the debugger to find lines which execute slow
 
 
 More detailed profiling
-=======================
+------------------------
 
 Occasionally this is not sufficient. In which case more detailed profiling is necessary
 

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/docs/developer/hyperion/index.rst

@@ -1,7 +1,7 @@
-General MX-Bluesky Developer Guide
+Hyperion Developer Guide
 ==================================
 
-Documentation is split into four categories, and each is also accessible from links in the side-bar.
+Documentation specific for the Hyperion module within MX-Bluesky
 
 .. grid:: 2
     :gutter: 2

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/docs/developer/hyperion/reference/param_hierarchy.puml

@@ -31,7 +31,7 @@ package Experiments {
     class RobotLoadAndEnergyChange
     class RobotLoadThenCentre
     class SpecifiedGridScan
-    class ThreeDGridScan
+    class HyperionThreeDGridScan
 }
 
 class HyperionParameters
@@ -66,9 +66,9 @@ RobotLoadThenCentre *-- RobotLoadAndEnergyChange
 RobotLoadThenCentre *-- PinTipCentreThenXrayCentre
 GridCommon <|-- SpecifiedGridScan
 WithScan <|-- SpecifiedGridScan
-SpecifiedGridScan <|-- ThreeDGridScan
-SplitScan <|-- ThreeDGridScan
-WithOptionalEnergyChange <|-- ThreeDGridScan
+SpecifiedGridScan <|-- HyperionThreeDGridScan
+SplitScan <|-- HyperionThreeDGridScan
+WithOptionalEnergyChange <|-- HyperionThreeDGridScan
 WithOavCentring <|-- GridCommon
 WithScan <|-- RotationScan
 RotationScanPerSweep <|-- RotationScan

mx_bluesky-1.4.1/docs/index.rst

@@ -0,0 +1,22 @@
+:html_theme.sidebar_secondary.remove:
+
+.. include:: ../README.rst
+    :end-before: when included in index.rst
+
+How the documentation is structured
+-----------------------------------
+
+The documentation consists of a Developer Guide for MX-Bluesky, which also includes documentation for Hyperion. At its current stage in development, there is no user guide.
+
+.. grid:: 1
+
+    .. grid-item-card:: :material-regular:`code;4em`
+        :link: developer/index
+        :link-type: doc
+
+        The Developer Guide contains documentation on how to install, develop and contribute changes to MX-Bluesky; explanations about the purpose of the software and some of the technical concepts; as well as how-to guides and tutorials.
+
+.. toctree::
+    :hidden:
+
+    developer/index

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/pyproject.toml

@@ -44,9 +44,9 @@ dependencies = [
     "blueapi >= 0.5.0",
     "daq-config-server >= 0.1.1",
     "ophyd == 1.9.0",
-    "ophyd-async >= 0.3a5",
+    "ophyd-async >= 0.8a5",
     "bluesky >= 1.13.0a4",
-    "dls-dodal == 1.35.0",
+    "dls-dodal == 1.36.2",
 ]
 
 

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/run_hyperion.sh

@@ -99,6 +99,8 @@ if [[ $START == 1 ]]; then
     module unload controls_dev
     module load dials
 
+    RELATIVE_SCRIPT_DIR=$( dirname -- "$0"; )
+    cd ${RELATIVE_SCRIPT_DIR}
 
     if [ -z "$LOG_DIR" ]; then
         if [ $IN_DEV == true ]; then

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/src/mx_bluesky/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.0'
-__version_tuple__ = version_tuple = (1, 4, 0)
+__version__ = version = '1.4.1'
+__version_tuple__ = version_tuple = (1, 4, 1)

mx_bluesky-1.4.1/src/mx_bluesky/beamlines/i04/redis_to_murko_forwarder.py

@@ -0,0 +1,178 @@
+import io
+import json
+import pickle
+from typing import TypedDict
+
+import numpy as np
+import zmq
+from dodal.beamlines.i04 import MURKO_REDIS_DB, REDIS_HOST, REDIS_PASSWORD
+from numpy.typing import NDArray
+from PIL import Image
+from redis import StrictRedis
+
+from mx_bluesky.common.utils.log import LOGGER
+
+MURKO_ADDRESS = "tcp://i04-murko-prod.diamond.ac.uk:8008"
+
+
+class MurkoRequest(TypedDict):
+    to_predict: NDArray
+    model_img_size: tuple[int, int]
+    save: bool
+    min_size: int
+    description: list
+    prefix: list[str]
+
+
+def get_image_size(image: NDArray) -> tuple[int, int]:
+    """Returns the width and height of a numpy image"""
+    return image.shape[1], image.shape[0]
+
+
+def send_to_murko_and_get_results(request: MurkoRequest) -> dict:
+    LOGGER.info(f"Sending {request['prefix']} to murko")
+    context = zmq.Context()
+    socket = context.socket(zmq.REQ)
+    socket.connect(MURKO_ADDRESS)
+    socket.send(pickle.dumps(request))
+    raw_results = socket.recv()
+    results = pickle.loads(raw_results)
+    LOGGER.info(f"Got {len(results['descriptions'])} results")
+    return results
+
+
+def correlate_results_to_uuids(request: MurkoRequest, murko_results: dict) -> list:
+    results = []
+    uuids = request["prefix"]
+
+    width, height = get_image_size(request["to_predict"][0])
+
+    for uuid, prediction in zip(uuids, murko_results["descriptions"], strict=False):
+        coords = prediction["most_likely_click"]
+        y_coord = coords[0] * height
+        x_coord = coords[1] * width
+        results.append(
+            {"uuid": uuid, "x_pixel_coord": x_coord, "y_pixel_coord": y_coord}
+        )
+    return results
+
+
+class BatchMurkoForwarder:
+    def __init__(self, redis_client: StrictRedis, batch_size: int):
+        """
+        Holds image data streamed from redis and forwards it to murko when:
+            * A set number have been received
+            * The shape of the images changes
+            * When `flush` is called
+
+        Once data has been forwarded this will then wait on the results and put them
+        back in redis.
+
+        Args:
+            redis_client: The client to send murko results back to redis.
+            batch_size: How many results to accumulate until they are flushed to redis.
+        """
+        self.redis_client = redis_client
+        self.batch_size = batch_size
+        self._uuids_and_images: dict[str, NDArray] = {}
+        self._last_image_size: tuple[int, int] | None = None
+        self._last_sample_id = ""
+
+    def _handle_batch_of_images(self, sample_id, images, uuids):
+        request_arguments: MurkoRequest = {
+            "model_img_size": (256, 320),
+            "to_predict": np.array(images),
+            "save": False,
+            "min_size": 64,
+            "description": [
+                "foreground",
+                "crystal",
+                "loop_inside",
+                "loop",
+                ["crystal", "loop"],
+                ["crystal", "loop", "stem"],
+            ],
+            "prefix": uuids,
+        }
+        predictions = send_to_murko_and_get_results(request_arguments)
+        results = correlate_results_to_uuids(request_arguments, predictions)
+        self._send_murko_results_to_redis(sample_id, results)
+
+    def _send_murko_results_to_redis(self, sample_id: str, results: list):
+        for result in results:
+            self.redis_client.hset(
+                f"murko:{sample_id}:results", result["uuid"], json.dumps(result)
+            )
+        self.redis_client.publish("murko-results", json.dumps(results))
+
+    def add(self, sample_id: str, uuid: str, image: NDArray):
+        """Add an image to the batch to send to murko."""
+        image_size = get_image_size(image)
+        self._last_sample_id = sample_id
+        if self._last_image_size and self._last_image_size != image_size:
+            self.flush()
+        self._uuids_and_images[uuid] = image
+        self._last_image_size = image_size
+        if len(self._uuids_and_images.keys()) >= self.batch_size:
+            self.flush()
+
+    def flush(self):
+        """Flush the batch to murko."""
+        if self._uuids_and_images:
+            self._handle_batch_of_images(
+                self._last_sample_id,
+                list(self._uuids_and_images.values()),
+                list(self._uuids_and_images.keys()),
+            )
+        self._uuids_and_images = {}
+        self._last_image_size = None
+
+
+class RedisListener:
+    TIMEOUT_S = 2
+
+    def __init__(
+        self,
+        redis_host=REDIS_HOST,
+        redis_password=REDIS_PASSWORD,
+        db=MURKO_REDIS_DB,
+        redis_channel="murko",
+    ):
+        self.redis_client = StrictRedis(
+            host=redis_host,
+            password=redis_password,
+            db=db,
+        )
+        self.pubsub = self.redis_client.pubsub()
+        self.channel = redis_channel
+        self.forwarder = BatchMurkoForwarder(self.redis_client, 10)
+
+    def _get_and_handle_message(self):
+        message = self.pubsub.get_message(timeout=self.TIMEOUT_S)
+        if message and message["type"] == "message":
+            data = json.loads(message["data"])
+            LOGGER.info(f"Received from redis: {data}")
+            uuid = data["uuid"]
+            sample_id = data["sample_id"]
+
+            # Images are put in redis as raw jpeg bytes, murko needs numpy arrays
+            raw_image = self.redis_client.hget(f"murko:{sample_id}:raw", uuid)
+            assert isinstance(raw_image, bytes)
+            image = Image.open(io.BytesIO(raw_image))
+            image = np.asarray(image)
+
+            self.forwarder.add(sample_id, uuid, image)
+
+        elif not message:
+            self.forwarder.flush()
+
+    def listen_for_image_data_forever(self):
+        self.pubsub.subscribe(self.channel)
+
+        while True:
+            self._get_and_handle_message()
+
+
+if __name__ == "__main__":
+    client = RedisListener()
+    client.listen_for_image_data_forever()

{mx_bluesky-1.4.0 → mx_bluesky-1.4.1}/src/mx_bluesky/beamlines/i04/thawing_plan.py

@@ -3,7 +3,7 @@ from collections.abc import Callable
 import bluesky.plan_stubs as bps
 import bluesky.preprocessors as bpp
 from bluesky.preprocessors import run_decorator, subs_decorator
-from dls_bluesky_core.core import MsgGenerator
+from bluesky.utils import MsgGenerator
 from dodal.beamlines.i04 import MURKO_REDIS_DB, REDIS_HOST, REDIS_PASSWORD
 from dodal.common import inject
 from dodal.devices.oav.oav_detector import OAV