mx-bluesky 1.5.5__tar.gz → 1.5.7__tar.gz

mx_bluesky-1.5.7/.devcontainer/devcontainer.json

@@ -0,0 +1,75 @@
+// For format details, see https://containers.dev/implementors/json_reference/
+{
+    "name": "Python 3 Developer Container",
+    "build": {
+        "dockerfile": "../Dockerfile",
+        "target": "developer"
+    },
+    "remoteEnv": {
+        // Allow X11 apps to run inside the container
+        "DISPLAY": "${localEnv:DISPLAY}"
+    },
+    "customizations": {
+        "vscode": {
+            // Set *default* container specific settings.json values on container create.
+            "settings": {
+                "python.defaultInterpreterPath": "/venv/bin/python"
+            },
+            // Add the IDs of extensions you want installed when the container is created.
+            "extensions": [
+                "ms-python.python",
+                "github.vscode-github-actions",
+                "tamasfe.even-better-toml",
+                "redhat.vscode-yaml",
+                "ryanluker.vscode-coverage-gutters",
+                "charliermarsh.ruff",
+                "ms-azuretools.vscode-docker"
+            ],
+            "python.testing.pytestArgs": [
+                "--cov=mx_bluesky",
+                "--cov-report",
+                "xml:cov.xml"
+            ],
+            "python.testing.unittestEnabled": false,
+            "python.testing.pytestEnabled": true,
+            "python.defaultInterpreterPath": "/venv/bin/python",
+            "editor.formatOnSave": true,
+            "editor.codeActionsOnSave": {
+                "source.organizeImports": "explicit"
+            },
+            "files.insertFinalNewline": true,
+            "[python]": {
+                "editor.rulers": [
+                    88
+                ],
+                "editor.codeActionsOnSave": {
+                    "source.fixAll.ruff": "explicit",
+                    "source.organizeImports.ruff": "explicit"
+                },
+                "editor.defaultFormatter": "charliermarsh.ruff"
+            },
+            "terminal.integrated.gpuAcceleration": "off",
+            "python.analysis.typeCheckingMode": "basic"
+        }
+    },
+    "features": {
+        // Some default things like git config
+        // add in eternal history and other bash features
+        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1": {}
+    },
+    // Create the config folder for the bash-config feature
+    "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config",
+    "runArgs": [
+        // Allow the container to access the host X11 display and EPICS CA
+        "--net=host",
+        // Make sure SELinux does not disable with access to host filesystems like tmp
+        "--security-opt=label=disable"
+    ],
+    // Mount the parent as /workspaces so we can pip install peers as editable
+    "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
+    "mounts": [
+        "source=/dls_sw/apps/dials/latest/latest/modules/dlstbx/src/dlstbx/,target=/dls_sw/apps/dials/latest/latest/modules/dlstbx/src/dlstbx/,type=bind,consistency=cached"
+    ],
+    // After the container is created, install the python project in editable form
+    "postCreateCommand": "pip install $([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e '.[dev]' && pip install -e /workspaces/dodal[dev] && pre-commit install && ln -s /dls_sw/apps/dials/latest/latest/modules/dlstbx/src/dlstbx/ /venv/lib/python3.12/site-packages/dlstbx"
+}

mx_bluesky-1.5.7/.vscode/mx-bluesky-dev-container.code-workspace

@@ -0,0 +1,23 @@
+{
+    "folders": [
+        {
+            "path": ".."
+        },
+        {
+            "path": "../../dodal"
+        },
+    ],
+    "settings": {
+        "python.languageServer": "Pylance",
+        "[python]": {
+            "editor.defaultFormatter": "ms-python.black-formatter"
+        },
+        "python.defaultInterpreterPath": "venv/bin/python",
+        "python.analysis.extraPaths": [
+            "../dodal"
+        ],
+        "python.analysis.autoImportCompletions": true,
+        "terminal.integrated.gpuAcceleration": "off",
+        "esbonio.sphinx.confDir": ""
+    }
+}

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/.vscode/mx-bluesky.code-workspace

@@ -13,7 +13,9 @@
             "editor.defaultFormatter": "ms-python.black-formatter"
         },
         "python.defaultInterpreterPath": ".venv/bin/python",
-        "python.analysis.extraPaths": ["../dodal"],
+        "python.analysis.extraPaths": [
+            "../dodal"
+        ],
         "python.analysis.autoImportCompletions": true,
         "terminal.integrated.gpuAcceleration": "off",
         "esbonio.sphinx.confDir": ""

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mx-bluesky
-Version: 1.5.5
+Version: 1.5.7
 Summary: Bluesky tools for MX Beamlines at DLS
 Author-email: Dominic Oram <dominic.oram@diamond.ac.uk>
 License:                                  Apache License
@@ -241,7 +241,7 @@ Requires-Dist: blueapi>=0.15.0
 Requires-Dist: ophyd>=1.10.5
 Requires-Dist: ophyd-async>=0.10.0a2
 Requires-Dist: bluesky>=1.13.1
-Requires-Dist: dls-dodal==1.57.0
+Requires-Dist: dls-dodal==1.59.1
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: build; extra == "dev"

mx_bluesky-1.5.7/docs/developer/general/how-to/get-started.md

@@ -0,0 +1,33 @@
+# Get Started with mx-bluesky
+
+## Development Environment
+
+We recommend using devcontainers with vscode as the IDE when developing using a DLS machine. Here's how you can set this up:
+
+- Clone this repo using SSH: `git clone git@github.com:DiamondLightSource/mx-bluesky.git`.
+- At the same directory level as you were in for step 1, clone dodal using SSH: `git clone git@github.com:DiamondLightSource/dodal.git`.
+You should now have a directory structure looking like`/some_path/mx-bluesky` and `/some_path/dodal`.
+- In a terminal, move to the mx-bluesky directory and open vscode:
+
+```
+cd /some_path/mx-bluesky  
+module load vscode  
+code .
+```
+
+- Make sure you have the [devcontainers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) installed in vscode.
+- In vscode, press `shift+ctrl+p`, and type "Dev Containers: Open Folder in Container", then select the mx-bluesky folder.
+- Once this has built, in vscode at the top right, click `File -> Open workspace from file`, then select `mx-bluesky/.vscode/mx-bluesky-dev-container.code-workspace`.
+- This should get dodal and mx-bluesky opened up in your workspace with all the correct settings. To prove to yourself that the environment is correct, type `tox -e tests` in a terminal in your devcontainer while inside the dodal and inside the mx-bluesky directory.
+
+### Notes
+- The devcontainer initially takes quite a long time to build. It will be much faster every other time, and only needs to be rebuild if we change the container environment.
+- The first time you make a commit after building, you will need to follow some git prompts to set your git name and email.
+- The old `dls_dev_env.sh` script creates a virtual environment in `mx-bluesky/.venv`, while the devcontainer creates one in `mx-bluesky/venv`. This is the only reason why we need separate code-workspace files.
+
+## Supported Python versions
+
+
+As a standard for the python versions to support, we are using the [numpy deprecation policy](https://numpy.org/neps/nep-0029-deprecation_policy.html)
+
+Currently supported versions are: 3.11, 3.12.

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/docs/developer/hyperion/index.rst

@@ -13,6 +13,7 @@ Documentation specific for the Hyperion module within MX-Bluesky
             :maxdepth: 1
 
             reference/readme
+            reference/baton
             reference/param-hierarchy
             reference/coordinate-systems
             deploying-hyperion

mx_bluesky-1.5.7/docs/developer/hyperion/reference/baton.rst

@@ -0,0 +1,22 @@
+Hyperion Baton
+==============
+
+The Hyperion baton is controlled by two PVs; these are described under `Special PVs`_
+
+An approximate outline of how the various baton workflows operates is outlined in the flowcharts below:
+
+.. image:: ../../../images/417717258-afbd813c-8941-445c-9fdf-255c167453cf.png
+
+.. _Special PVs: ../../../user/hyperion/pvs.html
+
+
+GDA-side Baton Integration
+--------------------------
+
+Below is a diagram showing how baton-handling in GDA integrates the GDA baton with the Hyperion Baton PV, using the 
+``HyperionUDCRunner``.
+
+The diagram represents an interim stage; at some point ``GDAUDCRunner`` can be eliminated and the baton management 
+simplified.
+
+.. uml:: gda-baton.puml

mx_bluesky-1.5.7/docs/developer/hyperion/reference/gda-baton.puml

@@ -0,0 +1,192 @@
+@startuml
+title GDA Baton Management
+
+package Server {
+    class Thread
+    class JythonServerThread extends Thread {
+        authorisationLevel: int
+        int getAuthorisationLevel()
+    }
+        
+    interface IBatonManager {
+        ClientDetails getClientInformation((myJSFIdentifier)
+        boolean requestBaton(uniqueIdentifier)
+        void returnBaton((uniqueIdentifier)
+        void assignBaton(myJSFIdentifier, indexOfReceiver, indexOfPasser)
+        boolean isBatonHeld()
+        boolean amIBatonHolder(myJSFIdentifier)
+        int effectiveAuthorisationLevelOf(uniqueId)
+        void addFacade(uniqueId, info: ClientDetails)
+        void removeFacade(uniqueId)
+    }
+    
+    class BatonManager implements IBatonManager {
+        useBaton = true
+        useRBAC = true
+        firstClientTakesBaton = true
+        batonHolder: String
+        facadeNames: Map<String, ClientInfo>
+        leaseHolders: Map<String, Long>
+    }
+    
+    note "Each client has a facade name aka uniqueId aka jsfIdentifier\n\
+There is a LeaseRefresher that periodically sends a BatonLeaseRenewRequest\n\
+to all clients, any interaction with the BatonManager extends the lease.\n\n\
+On first client registration, if firstClientTakesBaton is true then it will\n\
+assume the baton if it is not the server or an automated client." as batonManagerNote
+
+    BatonManager .. batonManagerNote
+
+
+    interface IJythonServerNotifier {
+        void notifyServer(source, data)    
+    }
+    interface LocalJython
+    class JythonServer implements LocalJython {
+        batonManager: BatonManager
+    }
+
+    note "JythonServer registers a client with the BatonManager via forwarding of addFacade()\n\
+The authorisationLevel is determined by asking the Authoriser" as jythonServerNote
+
+    JythonServer .. jythonServerNote
+        
+    interface Authoriser {
+        int getAuthorisationLevel(username)
+        int getDefaultPermissions(username)
+        isLocalStaff(username)    
+    }
+    
+    class FileAuthoriser implements Authoriser
+    note "Gets the permissions from user_permissions.xml" as fileAuthoriserNote
+    FileAuthoriser .. fileAuthoriserNote
+    class LdapAuthoriser extends FileAuthoriser
+    class SingleFileAuthoriser implements Authoriser
+    note "Authorisation method is determined by gda.gui.AcquisitionGUI.authorisationMethod" as authoriserNote
+    
+    Authoriser .. authoriserNote    
+
+    together {
+        class UDCBatonHandler {
+            runner: IUDCRunner
+        }
+        interface IUDCRunner
+        class GDAUDCRunner implements IUDCRunner
+        class HyperionUDCRunner implements IUDCRunner
+    }
+
+    note "\
+Creates the UDC Client by calling getUnattendedClientFacade() during configuration.\n\
+Creates the UDC Runner according to the gda.mx.udc.hyperion.enable property.\
+" as udcBatonHandlerNote
+    UDCBatonHandler .. udcBatonHandlerNote
+
+    class Device {
+        protectionLevel: int
+    }
+    class DeviceInterceptor
+    note "DeviceInterceptor is an interceptor on specific methods of Device, if the current thread\n\
+has an authorisationLevel below the protectionLevel of the device, and\n\
+the call is made from Jython the method call throws AccessDeniedException." as deviceNote
+    Device .. deviceNote
+    
+    
+    
+    class UDCCompleteEvent
+    
+}
+
+package Common {
+    together {
+        class BatonChanged
+        class BatonRequested
+        class BatonLeaseRenewRequest
+    }
+
+    interface Jython {
+        boolean amIBatonHolder(myJSFIdentifier)
+        int addFacade(jsfIdentifier, hostname, username, fullName, visitId)
+        boolean isBatonHeld()
+    }
+
+    class ClientInfo {
+        index: int
+        userId: String
+        fullName: String
+        hostname: String
+        authorisationLevel: int
+        visitID: String
+        
+        boolean isServer()
+        boolean isAutomatedUser()
+    }
+    note "isServer and isAutomatedUser are determined by the authorisationLevel" as clientInfoNote
+    ClientInfo .. clientInfoNote
+    
+    class ClientDetails extends ClientInfo {
+     hasBaton: boolean
+    }
+
+    interface IAuthorisationHolder {
+        int getAuthorisationLevel()
+        int getAuthorisationLevelAtRegistration()
+    }
+    
+    interface IBatonStateProvider {
+        boolean amIBatonHolder()
+        boolean isBatonHeld()
+        void addBatonChangedObserver(observer)
+        void deleteBatonChangedObserver(observer)
+    }
+    note "Access is enforced in the client via calls to amIBatonHolder()" as iBatonStateProviderNote
+
+    class JythonServerFacade implements IBatonStateProvider, IAuthorisationHolder {
+        commandServer: Jython
+        visitID: String
+        {static} JythonServerFacade getUnattendedClientFacade()
+    }
+}
+
+package Client {
+  class ApplicationWorkbenchAdvisor
+    note "ApplicationWorkbenchAdvisor receives the BatonLeaseRenewRequest and uses IBatonStateProvider\n\
+to renew the lease by calling amIBatonHolder()" as applicationWorkbenchAdvisorNote
+}
+
+'Server -up-> Common
+'Common -up[hidden]-> Client
+ApplicationWorkbenchAdvisor ..[norank] applicationWorkbenchAdvisorNote
+   
+ApplicationWorkbenchAdvisor --down> IBatonStateProvider
+
+IBatonStateProvider .. iBatonStateProviderNote
+LocalJython -up-|> Jython
+LocalJython --|> IJythonServerNotifier
+
+JythonServerFacade o-> Jython
+JythonServer o->BatonManager
+JythonServer --> Authoriser
+
+JythonServer --> JythonServerThread
+
+BatonChanged --> UDCBatonHandler : handles
+BatonRequested --> UDCBatonHandler : handles
+UDCCompleteEvent --> UDCBatonHandler : handles
+
+UDCBatonHandler -[norank]-> JythonServerFacade
+UDCBatonHandler o-> IUDCRunner
+BatonManager -up-> BatonChanged : fires
+BatonManager -up-> BatonRequested : fires
+BatonManager -up-> BatonLeaseRenewRequest : fires
+GDAUDCRunner -[norank]-> UDCCompleteEvent : fires
+BatonManager *-[norank]-> ClientInfo
+
+UDCCompleteEvent --> GDAUDCRunner : handles
+HyperionUDCRunner --> UDCCompleteEvent : fires
+
+DeviceInterceptor --> Device
+DeviceInterceptor --> JythonServerThread
+
+BatonManager -up-> IJythonServerNotifier
+
+@enduml

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/docs/developer/hyperion/reference/gridscan.puml

@@ -97,13 +97,13 @@ class GridScanParamsCommon {
     z2_start_mm
 }
 class PandAGridScanParams
-class ZebraGridScanParams
+class ZebraGridScanParamsThreeD
 
 AbstractExperimentBase <|-- AbstractExperimentWithBeamParams
 AbstractExperimentWithBeamParams <|-- GridScanParamsCommon
 GridScanParamsCommon <|-- PandAGridScanParams
-GridScanParamsCommon <|-- ZebraGridScanParams
+GridScanParamsCommon <|-- ZebraGridScanParamsThreeD
 
-HyperionThreeDGridScan --> ZebraGridScanParams : generates
+HyperionThreeDGridScan --> ZebraGridScanParamsThreeD : generates
 HyperionThreeDGridScan --> PandAGridScanParams : generates
 @enduml

mx_bluesky-1.5.7/docs/developer/murko-integration/explanations/DrawioImages.rst

@@ -0,0 +1,13 @@
+Draw.io Images
+--------------
+
+When working with diagrams created using Draw.io, it's important to ensure that the images retain their embedded metadata. This metadata allows the diagrams to be re-uploaded to Draw.io later for editing.
+
+To preserve this metadata:
+
+- When exporting from Draw.io, use the "File > Export As > PNG (with embedded XML)" or SVG (with embedded content) option.
+- Make sure the "Include a copy of my diagram" or similar checkbox is selected during export.
+
+This ensures that the image isn't just a flat graphic—it includes the necessary data for Draw.io to reconstruct and edit the diagram later.
+
+Tip: There is a chance of losing the metadata though if not handled properly. Sharing the image through Slack for example will delete the metadata associated with the image. Instead, use something like e-mail.

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/docs/developer/murko-integration/explanations/architecture.rst

@@ -3,7 +3,7 @@ Murko Architecture
 
 The architecture of how Murko is integrated is still in flux but as of 07/08 the following has been tried on the beamline.
 
-.. image:: ../images/murko_setup.drawio.png
+.. image:: ../images/Murko.drawio.png
 
 The mx-bluesky code is deployed into the `beamline kubernetes cluster <https://k8s-i04.diamond.ac.uk/>`_ behind a `blueAPI <https://github.com/DiamondLightSource/blueapi>`_ REST interface. Alongside this an instance of murko is also deployed to the k8s cluster.
 

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/docs/developer/murko-integration/index.rst

@@ -16,6 +16,7 @@ Its integration at DLS, using Bluesky, is still a work in progress but this docu
             :maxdepth: 1
 
             explanations/architecture
+            explanations/DrawioImages
 
         +++
 

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/docs/user/hyperion/configuration.rst

@@ -1,6 +1,9 @@
 Configuration
 -------------
 
+Hyperion main configuration
+===========================
+
 Configuration of several properties that control Hyperion execution are available. These can be edited in the 
 ``domain.properties`` file typically found in
 
@@ -8,6 +11,8 @@ Configuration of several properties that control Hyperion execution are availabl
 
     /dls_sw/<beamline>/software/daq_configuration/domain.properties
 
+Note that making a change to these properties generally requires a restart of the GDA server.
+
 
 .. csv-table:: Configuration properties
     :widths: auto
@@ -23,3 +28,27 @@ Configuration of several properties that control Hyperion execution are availabl
     "gda.mx.hyperion.xrc.use_gpu_results", "boolean", "If true, then zocalo gridscan processing uses the GPU results"
     "gda.mx.hyperion.xrc.use_roi_mode", "boolean", "If true then ROI mode is used."
     "gda.mx.udc.hyperion.enable", "boolean",  "Enables Hyperion UDC mode."
+
+Beamline configuration/calibration files
+========================================
+
+Hyperion makes use of other beamline configuration and calibration files which are also currently stored in 
+
+::
+
+    /dls_sw/<beamline>/software/daq_configuration
+
+These are currently shared with GDA, so changes to these files will affect both applications.
+
+Config Server
+=============
+
+If the `Config Server`_ is deployed and running, Hyperion is configured to use it in preference to reading the 
+``domain.properties`` file directly. However in the event of the config server being unavailable Hyperion will fall
+back to reading it from the filesystem.
+
+Note that currently the rest of the configuration files are not read from the config server, but the intention is that 
+ultimately it will be the source of all configuration and the remainder of the files in ``daq_configuration`` will be
+moved over to it.
+
+.. _Config Server: https://github.com/DiamondLightSource/daq-config-server/

mx_bluesky-1.5.7/docs/user/hyperion/index.rst

@@ -0,0 +1,57 @@
+Hyperion User Guide
+===================
+
+The Hyperion User Guide describes how to run, configure and troubleshoot Hyperion. For installation instructions, see
+the `Developer Guide`_.
+
+What is Hyperion?
+-----------------
+
+Hyperion is a service for running high throughput unattended data collection (UDC). It does not provide a user 
+interface, instead instructions are pulled from Agamemnon which is controlled by information obtained in ISPyB.
+
+The software supports two modes of operation:
+
+* UDC mode (experimental) where Hyperion automatically fetches instructions from Agamemnon.
+* GDA mode (where GDA fetches and decodes the Agamemnon
+  instructions). GDA mode will be removed in a future release.
+
+The mode of operation is determined by configuration using the ``gda.mx.udc.hyperion.enable`` parameter inside 
+GDA properties. See `configuration`_ for other properties that control Hyperions behavior. 
+
+Once Hyperion has received a request, either from GDA or directly from Agamemnon, it will do the following tasks:
+
+- Robot Load (if the requested sample has not yet been loaded)
+- Pin tip centring using the OAV
+- Xray Centring using 2 2D gridscans 
+- A number of data collections, depending on the number of centres returned from Zocalo and the applied selection criteria
+
+During this it will generate the following outputs:
+
+- Snapshots on robot load, XRC and rotations
+- Data collections in ISPyB for the gridscans and rotations as well as entries for robot load/unload
+- NeXus files for each data collection
+- Alert notifications on loading a new container and on beamline error conditions when intervention is required.  
+
+To increase throughput, the behavior of Hyperion differs in many ways from UDC under GDA. The main ways this manifests 
+to a user is:
+
+- Xray Centring is 2 2D gridscans (rather than the one gridscan and one line scan in GDA). For speed, the eiger is only
+  armed once for both of these and the data is split up after the fact. This means on the filesystem there is one set of 
+  hdf files for both gridscans but two NeXus files. The two grid scans are stored in ISPyB under one data collection 
+  group so at first glance it may look like one gridscan in synchweb but you can click through to see both scans.
+- If an xray centre cannot be done on a pin (because it is too long/short or bent), or if no diffraction is found in the 
+  xray centring data, the sample will be immediately unloaded without a rotation scan being done and the next sample loaded.
+
+For a more detailed breakdown of the operation, it may be helpful to browse the  `Code Map`_
+
+.. _Code Map: ../../developer/code-map 
+.. _Developer Guide: ../../developer/hyperion/deploying-hyperion
+.. _configuration: ./configuration
+
+.. toctree::
+    :caption: Topics
+    :maxdepth: 1
+    :glob:
+
+    *

{mx_bluesky-1.5.5 → mx_bluesky-1.5.7}/docs/user/hyperion/pvs.rst

@@ -10,3 +10,12 @@ CURRENT_USER.
 Hyperion will only start operating once REQUESTED_USER has been set to Hyperion, at which point it will set 
 CURRENT_USER. To request the baton from Hyperion, set REQUESTED_USER to some other value, when Hyperion completes the
 current action, it will relinquish CURRENT_USER.
+
+Commissioning Mode
+~~~~~~~~~~~~~~~~~~
+
+The baton device also has a COMMISSIONING PV, which when the PV is set to 1, indicates that commissioning mode is 
+enabled.
+
+Commissioning mode enables testing of Hyperion when there is no beam. Beamline scientists should ensure that the 
+COMMISSIONING PV is set to 0 before running Hyperion in production.