mx-bluesky 1.5.1__tar.gz → 1.5.2__tar.gz

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/.github/workflows/_container.yml

@@ -42,18 +42,39 @@ jobs:
           file: Dockerfile.blueapi
           load: true
 
-      - name: Create tags for publishing image
-        id: meta
+      - name: Create tags for publishing Hyperion image
+        id: meta-hyperion
         uses: docker/metadata-action@v5
         with:
           images: |
             ghcr.io/${{ github.repository }}
+          labels: |
+            org.opencontainers.image.title=mx-bluesky
+            org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection
+          annotations: |
+            org.opencontainers.image.title=mx-bluesky
+            org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection
+          tags: |
+            type=ref,event=tag
+            type=raw,value=latest
+
+      - name: Create tags for publishing BlueAPI image
+        id: meta-blueapi
+        uses: docker/metadata-action@v5
+        with:
+          images: |
             ghcr.io/${{ github.repository }}-blueapi
+          labels: |
+            org.opencontainers.image.title=mx-bluesky-blueapi
+            org.opencontainers.image.description=BlueAPI plans, stubs and utilities for MX beamlines
+          annotations: |
+            org.opencontainers.image.title=mx-bluesky-blueapi
+            org.opencontainers.image.description=BlueAPI plans, stubs and utilities for MX beamlines
           tags: |
             type=ref,event=tag
             type=raw,value=latest
 
-      - name: Push cached image to container registry
+      - name: Push cached Hyperion image to container registry
         if: github.ref_type == 'tag'
         uses: docker/build-push-action@v6
         env:
@@ -64,10 +85,12 @@ jobs:
           context: .
           file: Dockerfile.hyperion
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-hyperion.outputs.tags }}
+          labels: ${{ steps.meta-hyperion.outputs.labels }}
+          # Set the annotations to make the description appear in the GH repo page
+          annotations: ${{ steps.meta-hyperion.outputs.annotations }}
 
-      - name: Push Dockerfile.blueapi image to container registry
+      - name: Push cached BlueAPI image to container registry
         if: github.ref_type == 'tag'
         uses: docker/build-push-action@v6
         env:
@@ -76,5 +99,7 @@ jobs:
           context: .
           file: Dockerfile.blueapi
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-blueapi.outputs.tags }}
+          labels: ${{ steps.meta-blueapi.outputs.labels }}
+          # Set the annotations to make the description appear in the GH repo page
+          annotations: ${{ steps.meta-blueapi.outputs.annotations }}

mx_bluesky-1.5.2/.github/workflows/_helm.yml

@@ -0,0 +1,23 @@
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install Helm
+        uses: Azure/setup-helm@v4.3.0
+        id: install
+
+      - name: Helm login to GHCR
+        if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io/${{ github.repository }} --username ${{ github.repository_owner }} --password-stdin
+
+      - name: Package and push chart
+        if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
+        run: |
+          helm dependencies update helm/mx-bluesky-blueapi
+          helm package helm/mx-bluesky-blueapi --version ${{ steps.meta.outputs.version }} --app-version ${{ steps.meta.outputs.version }} -d /tmp/
+          helm push /tmp/mx-bluesky-blueapi-${{ steps.meta.outputs.version }}.tgz oci://ghcr.io/diamondlightsource/charts

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/.github/workflows/ci.yml

@@ -38,6 +38,14 @@ jobs:
       contents: read
       packages: write
 
+  helm:
+    needs: container
+    if: needs.check.outputs.branch-pr == ''
+    uses: ./.github/workflows/_helm.yml
+    permissions:
+      contents: read
+      packages: write
+
   docs:
     needs: check
     if: needs.check.outputs.branch-pr == ''

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
         args: ["--maxkb=500"]
       - id: check-yaml
         args: ["--allow-multiple-documents"]
-        exclude: ^helmcharts/
+        exclude: ^helm/
       - id: check-merge-conflict
       - id: end-of-file-fixer
 

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/Dockerfile.blueapi

@@ -1,5 +1,7 @@
- # See https://github.com/DiamondLightSource/mx-bluesky/issues/1004
-FROM ghcr.io/diamondlightsource/blueapi:0.10.1
+# Before changing this BlueAPI version number, you should check that 
+# the BlueAPI pin in pyproject.toml is compatible, else the version will
+# be overridden
+FROM ghcr.io/diamondlightsource/blueapi:1.0.0-rc.2
 
 ARG DEBIAN_FRONTEND=noninteractive
 

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mx-bluesky
-Version: 1.5.1
+Version: 1.5.2
 Summary: Bluesky tools for MX Beamlines at DLS
 Author-email: Dominic Oram <dominic.oram@diamond.ac.uk>
 License:                                  Apache License
@@ -235,12 +235,12 @@ Requires-Dist: scipy
 Requires-Dist: semver
 Requires-Dist: deepdiff
 Requires-Dist: matplotlib
-Requires-Dist: blueapi>=0.11.1
+Requires-Dist: blueapi>=0.15.0
 Requires-Dist: daq-config-server==0.1.1
 Requires-Dist: ophyd>=1.10.5
 Requires-Dist: ophyd-async>=0.10.0a2
 Requires-Dist: bluesky>=1.13.1
-Requires-Dist: dls-dodal==1.51.0
+Requires-Dist: dls-dodal==1.52.0
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: build; extra == "dev"

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/docs/developer/hyperion/deploying-hyperion.rst

@@ -80,6 +80,16 @@ are not already present:
 * ``rmq-creds``
 * ``rmq-api-reader``
 
+In addition, an ispyb credentials secret must be created with the following name:
+* ``ispyb-hyperion-cfg``
+
+
+For creating the secrets, use ``kubectl create secret``:
+
+::
+
+    kubectl create secret -n i03-beamline generic ispyb-hyperion-cfg --from-file=ispyb.cfg=<path-to-ispyb-hyperion-ixx.cfg>
+
 Production deployment
 ~~~~~~~~~~~~~~~~~~~~~
 

{mx_bluesky-1.5.1/helmcharts → mx_bluesky-1.5.2/helm}/hyperion/templates/deployment.yaml

@@ -68,48 +68,13 @@ spec:
                 name: {{ .secretName }}
             {{- end -}} {{/* range */}}
             {{- end -}} {{/* if */}}
+        - name: ispyb-secrets
+          secret:
+            type: secret
+            secretName: ispyb-hyperion-cfg
       containers:
       - name: hyperion
-        image: {{ .Values.application.imageRepository}}/mx-bluesky:{{ .Values.application.appVersion }}
-        securityContext:
-          capabilities:
-            # Required for hostNetwork 
-            drop:
-              - AUDIT_WRITE
-              - CHOWN
-              - DAC_OVERRIDE
-              - FOWNER
-              - FSETID
-              - KILL
-              - MKNOD
-              - NET_BIND_SERVICE
-              - NET_RAW
-              - SETFCAP
-              - SETGID
-              - SETPCAP
-              - SETUID
-              - SYS_CHROOT
-              - AUDIT_CONTROL
-              - BLOCK_SUSPEND
-              - DAC_READ_SEARCH
-              - IPC_LOCK
-              - IPC_OWNER
-              - LEASE
-              - LINUX_IMMUTABLE
-              - MAC_ADMIN
-              - MAC_OVERRIDE
-              - SYS_ADMIN
-              - SYS_BOOT
-              - SYS_MODULE
-              - SYS_NICE
-              - SYS_PACCT
-              - SYS_MODULE
-              - SYS_PTRACE
-              - SYS_RESOURCE
-              - SYS_TIME
-              - SYS_TTY_CONFIG
-              - SYSLOG
-              - WAKE_ALARM            
+        image: {{ .Values.application.imageRepository}}/mx-bluesky:{{ .Values.application.appVersion }}    
         resources:
           limits:
             cpu: "1"
@@ -133,7 +98,7 @@ spec:
           - name: ZOCALO_CONFIG
             value: "/zocalo/config/configuration.yaml"
           - name: ISPYB_CONFIG_PATH
-            value: "/dls_sw/dasc/mariadb/credentials/ispyb-hyperion-{{ .Values.application.beamline }}.cfg"
+            value: "/ispyb/secrets/ispyb.cfg"
           {{- end }}
         readinessProbe:
           exec:
@@ -166,4 +131,6 @@ spec:
             name: zocalo-config
           - mountPath: "/zocalo/secrets"
             name: zocalo-secrets
+          - mountPath: "/ispyb/secrets"
+            name: ispyb-secrets
       hostNetwork: true

{mx_bluesky-1.5.1/helmcharts → mx_bluesky-1.5.2/helm}/hyperion/values.yaml

@@ -9,7 +9,7 @@ application:
   beamline: i03
   dev: false
   logDir: "/dls_sw/i03/logs/bluesky/hyperion-k8s"
-  debugLogDir: "/dls/tmp/i03/logs/bluesky"
+  debugLogDir: "/dls/tmp/i03/logs/bluesky-k8s"
   dataDir: "/dls/i03/data"
   # These should be overridden at install time
   projectDir: SET_ON_INSTALL

mx_bluesky-1.5.2/helm/mx-bluesky-blueapi/Chart.yaml

@@ -0,0 +1,16 @@
+apiVersion: v2
+name: mx-bluesky-blueapi
+description: A helm chart deploying a worker pod that runs Bluesky plans
+
+# This chart provides extra configuration to a BlueAPI application which is generic to all MX beamlines,
+# and does not startup an application by itself. A BlueAPI service in ixx-services should add this chart
+# as a dependency. It currently mounts zocalo and ispyb credentials as config maps, with the assumption 
+# that the beamline cluster have these secrets available. In the future, this helmchart may change to a 
+# library type
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+# This is updated automatically by CI, the appVersion is automatically inserted by CI
+version: 0.1.1

mx_bluesky-1.5.2/helm/mx-bluesky-blueapi/templates/ispyb-config.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Release.Name }}-ispyb-config"
+  namespace: "{{ .Release.Namespace }}"
+  labels:
+    app.kubernetes.io/name: ispyb-config
+    app.kubernetes.io/instance: "{{ .Release.Name }}-ispyb-config"
+    app.kubernetes.io/version: "{{ .Chart.Version }}"
+    app.kubernetes.io/part-of: mx-bluesky-blueapi 
+data:
+  configuration.yaml: |-
+    version: 1
+  {{- if .Values.ispyb.secrets }}
+  {{- range .Values.ispyb.secrets }}
+    {{ .name }}: "/ispyb/secrets/{{ .fileName }}"
+  {{- end -}} {{/* range */}}
+  {{- end -}} {{/* if */}}

mx_bluesky-1.5.2/helm/mx-bluesky-blueapi/templates/zocalo-config.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "zocalo-config"
+  namespace: "{{ .Release.Namespace }}"
+  labels:
+    app.kubernetes.io/name: zocalo-config
+    app.kubernetes.io/instance: "{{ .Release.Name }}-zocalo-config"
+    app.kubernetes.io/version: "{{ .Chart.Version }}"
+    app.kubernetes.io/part-of: mx-bluesky-blueapi 
+data:
+  configuration.yaml: |-
+    version: 1
+    graylog:
+      plugin: graylog
+      {{- .Values.zocalo.graylog | toYaml | nindent 6 }}
+  {{- if .Values.zocalo.secrets }}
+  {{- range .Values.zocalo.secrets }}
+    {{ .name }}: "/zocalo/secrets/{{ .fileName }}"
+  {{- end -}} {{/* range */}}
+  {{- end -}} {{/* if */}}
+
+
+    default-transport:
+      plugin: transport
+      default: {{ .Values.zocalo.defaultTransport }}
+
+    environments:
+
+      bluesky:
+        logging: graylog
+        plugins:
+          - default-transport
+        {{- if .Values.zocalo.secrets }}
+        {{- range .Values.zocalo.secrets }}
+          - {{ .name }}
+        {{- end -}} {{/* range */}}
+        {{- end -}} {{/* if */}}

mx_bluesky-1.5.2/helm/mx-bluesky-blueapi/values.yaml

@@ -0,0 +1,19 @@
+zocalo:
+  defaultTransport: "PikaTransport"
+  graylog:
+    host: graylog-log-target.diamond.ac.uk
+    port: 12208
+    protocol: UDP
+  secrets:
+    - name: rabbitmq
+      secretName: rmq-creds
+      fileName: rabbitmq-credentials.yml
+    - name: rabbitmq-api-reader
+      secretName: rmq-api-reader
+      fileName: rabbitmq-api-reader.yml
+
+ispyb:
+  secrets:
+  - name: ispyb-creds
+    secretName: ispyb-creds
+    fileName: ispyb-credentials.yml

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/pyproject.toml

@@ -42,12 +42,12 @@ dependencies = [
     # These dependencies may be issued as pre-release versions and should have a pin constraint
     # as by default pip-install will not upgrade to a pre-release.
     #
-    "blueapi >= 0.11.1",
+    "blueapi >= 0.15.0",
     "daq-config-server == 0.1.1",
     "ophyd >= 1.10.5",
     "ophyd-async >= 0.10.0a2",
     "bluesky >= 1.13.1",
-    "dls-dodal == 1.51.0",
+    "dls-dodal == 1.52.0",
 ]
 
 
@@ -248,5 +248,15 @@ modules = "mx_bluesky.beamlines.*"
 [[tool.importlinter.contracts]]
 name = "Beamlines do not import from hyperion"
 type = "forbidden"
-source_modules = "mx_bluesky.beamlines.*"
-forbidden_modules = "mx_bluesky.hyperion"
+source_modules = "mx_bluesky.beamlines.**"
+forbidden_modules = "mx_bluesky.hyperion.**"
+
+[[tool.importlinter.contracts]]
+name = "Only Hyperion and i04 can import from phase1_zebra"
+type = "forbidden"
+source_modules = ["mx_bluesky.beamlines.**", "mx_bluesky.common.**"]
+forbidden_modules = ["mx_bluesky.phase1_zebra.**"]
+ignore_imports = [
+    "mx_bluesky.beamlines.i04.** -> mx_bluesky.phase1_zebra.**",
+    "mx_bluesky.hyperion.** -> mx_bluesky.phase1_zebra.**",
+]

{mx_bluesky-1.5.1 → mx_bluesky-1.5.2}/src/mx_bluesky/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.5.1'
-__version_tuple__ = version_tuple = (1, 5, 1)
+__version__ = version = '1.5.2'
+__version_tuple__ = version_tuple = (1, 5, 2)

mx_bluesky-1.5.2/src/mx_bluesky/beamlines/i04/__init__.py

@@ -0,0 +1,6 @@
+from mx_bluesky.beamlines.i04.experiment_plans import (
+    i04_grid_detect_then_xray_centre_plan,
+)
+from mx_bluesky.beamlines.i04.thawing_plan import thaw, thaw_and_stream_to_redis
+
+__all__ = ["thaw", "thaw_and_stream_to_redis", "i04_grid_detect_then_xray_centre_plan"]

mx_bluesky-1.5.2/src/mx_bluesky/beamlines/i04/callbacks/murko_callback.py

@@ -0,0 +1,110 @@
+import copy
+import json
+from datetime import timedelta
+from typing import TypedDict
+
+from bluesky.callbacks import CallbackBase
+from dodal.log import LOGGER
+from event_model.documents import Event, RunStart, RunStop
+from redis import StrictRedis
+
+
+class OmegaReading(TypedDict):
+    value: float
+    timestamp: float
+
+
+def extrapolate_omega(
+    latest_omega: OmegaReading, previous_omega: OmegaReading, image_timestamp: float
+) -> float:
+    """Extrapolate an image omega from previous omegas.
+
+    There are a number of assumptions in this calculation:
+    * The speed of the smargon is fixed
+    * The timestamps from the two different devices are synchronised and match the data
+      exactly
+
+    These are accepted to be reasonable based on larger errors likely coming from murko
+    itself and that the results ultimately will be averaged out.
+    """
+    omega_per_sec = (latest_omega["value"] - previous_omega["value"]) / (
+        latest_omega["timestamp"] - previous_omega["timestamp"]
+    )
+    time_elapsed = image_timestamp - latest_omega["timestamp"]
+    return latest_omega["value"] + time_elapsed * omega_per_sec
+
+
+class MurkoCallback(CallbackBase):
+    """A callback that triggers murko processing of images.
+
+    It combines metadata readings from e.g the goniometer rotation with the uuid's given
+    to us by an `OAVToRedisForwarder` (which describe the location of images in redis).
+    And writes these as a package to redis. A separate service then forwards this to murko.
+
+    The metadata and image data arrive independently, it is expected that the image data
+    is arriving at a faster rate than gonio metadata and so the value of omega for when
+    the image arrives is extrapolated based on previous omega readings.
+    """
+
+    DATA_EXPIRY_DAYS = 7
+
+    def __init__(self, redis_host: str, redis_password: str, redis_db: int = 0):
+        self.redis_client = StrictRedis(
+            host=redis_host, password=redis_password, db=redis_db
+        )
+        self.last_uuid = None
+        self.previous_omegas: list[OmegaReading] = []
+
+    def start(self, doc: RunStart) -> RunStart | None:
+        self.sample_id = doc.get("sample_id")
+        self.murko_metadata = {
+            "zoom_percentage": doc.get("zoom_percentage"),
+            "microns_per_x_pixel": doc.get("microns_per_x_pixel"),
+            "microns_per_y_pixel": doc.get("microns_per_y_pixel"),
+            "beam_centre_i": doc.get("beam_centre_i"),
+            "beam_centre_j": doc.get("beam_centre_j"),
+            "sample_id": self.sample_id,
+        }
+        self.last_uuid = None
+        self.previous_omegas = []
+        LOGGER.info(f"Starting to stream metadata to murko under {self.sample_id}")
+        return doc
+
+    def event(self, doc: Event) -> Event:
+        if latest_omega := doc["data"].get("smargon-omega"):
+            if len(self.previous_omegas) <= 2 and self.last_uuid:
+                # For the first few images there's not enough data to extrapolate so we
+                # match them one to one
+                self.call_murko(self.last_uuid, latest_omega)
+            self.previous_omegas.append(
+                OmegaReading(
+                    value=latest_omega,
+                    timestamp=doc["timestamps"]["smargon-omega"],
+                )
+            )
+        elif (uuid := doc["data"].get("oav_to_redis_forwarder-uuid")) is not None:
+            if len(self.previous_omegas) >= 2:
+                omega = extrapolate_omega(
+                    self.previous_omegas[-1],
+                    self.previous_omegas[-2],
+                    doc["timestamps"]["oav_to_redis_forwarder-uuid"],
+                )
+                LOGGER.info(f"Using extrapolated omega of {omega}")
+                self.call_murko(uuid, omega)
+            self.last_uuid = uuid
+        return doc
+
+    def call_murko(self, uuid: str, omega: float):
+        metadata = copy.deepcopy(self.murko_metadata)
+        metadata["omega_angle"] = omega
+        metadata["uuid"] = uuid
+
+        # Send metadata to REDIS and trigger murko
+        redis_key = f"murko:{metadata['sample_id']}:metadata"
+        self.redis_client.hset(redis_key, uuid, json.dumps(metadata))
+        self.redis_client.expire(redis_key, timedelta(days=self.DATA_EXPIRY_DAYS))
+        self.redis_client.publish("murko", json.dumps(metadata))
+
+    def stop(self, doc: RunStop) -> RunStop | None:
+        LOGGER.info(f"Finished streaming {self.sample_id} to murko")
+        return doc