bec-server 4.4.2__tar.gz → 4.5.0__tar.gz

{bec_server-4.4.2 → bec_server-4.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: bec-server
-Version: 4.4.2
+Version: 4.5.0
 Summary: BEC server
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

{bec_server-4.4.2 → bec_server-4.5.0}/bec_server/scan_server/direct_scan_worker.py

@@ -79,6 +79,7 @@ class DirectScanWorker:
         scan.actions._update_queue_info_callback = self.update_queue_info
         queue = self.worker.current_instruction_queue_item
         try:
+            scan.actions._hook_broadcaster.start()
             with self.worker.device_manager._rpc_method(scan.actions.rpc_call):
                 scan.actions._initialize_scan()
                 for step in SCAN_SEQUENCE:
@@ -86,7 +87,8 @@ class DirectScanWorker:
                     if not method:
                         raise ScanAbortion(f"Scan is missing required method: {step}")
                     self.check_for_interruption()
-                    method()
+                    with scan.actions._scan_hook_context(step):
+                        method()
                     self.run_post_scan_step(step, scan)
         except ScanAbortion as exc:
             if not self._prepare_exception_cleanup(queue, exc):
@@ -97,7 +99,10 @@ class DirectScanWorker:
                 return
             self._handle_exception(exc)
         finally:
-            self._release_scan_locks(scan)
+            try:
+                scan.actions._hook_broadcaster.stop()
+            finally:
+                self._release_scan_locks(scan)
 
         if queue is None:
             return
@@ -245,7 +250,9 @@ class DirectScanWorker:
         try:
             scan._shutdown_event.clear()
             with self.worker.device_manager._rpc_method(scan.actions.rpc_call):
-                scan.on_exception(hook_exc)
+                # pylint: disable=protected-access
+                with scan.actions._scan_hook_context("on_exception"):
+                    scan.on_exception(hook_exc)
 
         except Exception:
             scan.actions.send_client_info("")

{bec_server-4.4.2 → bec_server-4.5.0}/bec_server/scan_server/scan_guard.py

@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING
 from bec_lib import messages
 from bec_lib.endpoints import MessageEndpoints
 from bec_lib.logger import bec_logger
-from bec_server.scan_server.scan_queue import ScanQueueStatus
 
 logger = bec_logger.logger
 
@@ -308,28 +307,21 @@ class ScanGuard:
             logger.error(f"Invalid queue: {target_queue}")
             self._send_scan_queue_order_change_response(False, f"Invalid queue: {target_queue}")
             return
-        if self.parent.queue_manager.queues[target_queue].status != ScanQueueStatus.PAUSED:
-            logger.error(f"Queue {target_queue} is not paused.")
-            self._send_scan_queue_order_change_response(
-                False, f"Queue {target_queue} is not paused. Cannot move scans."
-            )
-            return
-        if msg.action == "move_to" and msg.target_position is None:
-            logger.error("Missing target_position")
-            self._send_scan_queue_order_change_response(
-                False, "Missing target_position for move_to"
-            )
-            return
-
         queue = self.parent.queue_manager.queues[target_queue]
-        for scan in queue.queue:
-            if msg.scan_id in scan.scan_id:
-                break
-        else:
-            logger.error(f"Scan {msg.scan_id} not found in queue {target_queue}")
-            self._send_scan_queue_order_change_response(
-                False, f"Scan {msg.scan_id} not found in queue {target_queue}"
-            )
+        with queue._lock:  # pylint: disable=protected-access
+            active = queue.active_instruction_queue
+            if active is not None and msg.scan_id in active.scan_id:
+                rejection = "The running scan cannot be moved."
+            elif msg.action == "move_to" and msg.target_position is None:
+                rejection = "Missing target_position for move_to"
+            elif not any(msg.scan_id in scan.scan_id for scan in queue.queue):
+                rejection = f"Scan {msg.scan_id} not found in queue {target_queue}"
+            else:
+                rejection = None
+
+        if rejection is not None:
+            logger.error(rejection)
+            self._send_scan_queue_order_change_response(False, rejection)
             return
         self.device_manager.connector.send(sqoc, msg)
         self._send_scan_queue_order_change_response(True, "Order change accepted")

bec_server-4.5.0/bec_server/scan_server/scan_hook_broadcaster.py

@@ -0,0 +1,114 @@
+"""Broadcast and store the latest scan lifecycle or instruction update for clients."""
+
+from __future__ import annotations
+
+import threading
+from typing import TYPE_CHECKING
+
+from bec_lib import messages
+from bec_lib.endpoints import MessageEndpoints
+from bec_lib.logger import bec_logger
+
+if TYPE_CHECKING:
+    from bec_lib.redis_connector import RedisConnector
+
+logger = bec_logger.logger
+
+
+class ScanHookBroadcaster:
+    """Send only the latest update in each 0.1 second broadcast window, unless flushed at once."""
+
+    def __init__(self, connector: RedisConnector) -> None:
+        self._connector = connector
+        self._lock = threading.Lock()
+        self._publish_lock = threading.Lock()
+        self._latest_update: messages.ScanHookMessage | None = None
+        self._timer: threading.Timer | None = None
+        self._active = False
+        self._stopped = False
+
+    @property
+    def active(self) -> bool:
+        """Whether events are collected, i.e. the scan worker is executing the scan."""
+        with self._lock:
+            return self._active
+
+    def start(self) -> None:
+        """Enable event collection when the scan worker begins execution."""
+        with self._lock:
+            if self._stopped:
+                raise RuntimeError("A stopped scan hook broadcaster cannot be restarted.")
+            self._active = True
+
+    def stop(self) -> None:
+        """Stop collection, join the pending timer, and attempt final delivery."""
+        with self._lock:
+            if self._stopped:
+                return
+            self._stopped = True
+            self._active = False
+            timer = self._timer
+            if timer is not None:
+                timer.cancel()
+        if timer is not None and timer is not threading.current_thread():
+            timer.join()
+        self._flush_scheduled()
+
+    def add(self, event: messages.ScanHookMessage) -> None:
+        """Replace the pending update and schedule a flush when the first event arrives.
+
+        Args:
+            event (messages.ScanHookMessage): Lifecycle or instruction event to broadcast.
+        """
+        with self._lock:
+            if not self._active:
+                return
+            self._latest_update = event
+            if self._timer is None:
+                self._timer = threading.Timer(0.1, self._flush_scheduled)
+                self._timer.daemon = True
+                self._timer.start()
+
+    def _flush_scheduled(self) -> None:
+        try:
+            self.flush()
+        except Exception:  # pylint: disable=broad-except
+            logger.exception("Failed to broadcast scan hook events")
+
+    def flush_now(self) -> None:
+        """Send the pending update without waiting for the window, e.g. for a step change.
+
+        A failed publication is logged and does not interrupt the caller.
+        """
+        self._flush_scheduled()
+
+    def flush(self) -> None:
+        """Send the latest pending update and cancel the pending timer.
+
+        Scans call this on every exit path to deliver their final update.
+        Failed publications retain the update for retry unless a newer one arrives.
+        """
+        timer = None
+        try:
+            # Serialize publications without blocking producers on network I/O.
+            with self._publish_lock:
+                with self._lock:
+                    timer = self._timer
+                    self._timer = None
+                    if timer is not None:
+                        timer.cancel()
+                    event = self._latest_update
+                    self._latest_update = None
+                if event is None:
+                    return
+                try:
+                    self._connector.set_and_publish(MessageEndpoints.scan_hook(), event)
+                except Exception:
+                    with self._lock:
+                        if not self._stopped and self._latest_update is None:
+                            self._latest_update = event
+                    raise
+        finally:
+            # A timer may itself be waiting to publish, so release both locks first.
+            if timer is not None and timer is not threading.current_thread():
+                timer.join()

{bec_server-4.4.2 → bec_server-4.5.0}/bec_server/scan_server/scan_queue.py

@@ -237,14 +237,21 @@ class QueueManager:
             msg (messages.ScanQueueOrderMessage): ScanQueueOrderMessage
 
         """
-        with self._lock:
+        target_queue = msg.queue
+        # take the worker's queue lock first (same order as the worker) so the running item
+        # cannot be popped while we decide what may move
+        with self.queues[target_queue]._lock, self._lock:
             logger.info(f"Handling scan queue order change: {msg}")
-            target_queue = msg.queue
             queue = self.queues[target_queue].queue
             queue_item = self._get_queue_item_by_scan_id(msg)
             if not queue_item:
                 logger.error(f"Scan {msg.scan_id} not found in queue {target_queue}")
                 return
+            # in a running queue, the active item stays first and nothing moves in front of it
+            first = self._first_movable_index(target_queue)
+            if queue.index(queue_item) < first:
+                logger.error(f"Scan {msg.scan_id} is running and cannot be moved")
+                return
 
             if msg.action == "move_to":
                 # move the scan to the target position
@@ -252,7 +259,7 @@ class QueueManager:
                     logger.error("Missing target_position")
                     return
 
-                position = max(0, min(msg.target_position, len(queue) - 1))
+                position = max(first, min(msg.target_position, len(queue) - 1))
 
                 queue.remove(queue_item)
                 queue.insert(position, queue_item)
@@ -260,7 +267,7 @@ class QueueManager:
             if msg.action == "move_up":
                 # move the scan up by one position
                 idx = queue.index(queue_item)
-                if idx == 0:
+                if idx <= first:
                     return
                 queue.remove(queue_item)
                 queue.insert(idx - 1, queue_item)
@@ -276,7 +283,7 @@ class QueueManager:
             if msg.action == "move_top":
                 # move the scan to the top of the queue
                 queue.remove(queue_item)
-                queue.insert(0, queue_item)
+                queue.insert(first, queue_item)
 
             if msg.action == "move_bottom":
                 # move the scan to the bottom of the queue
@@ -285,6 +292,19 @@ class QueueManager:
 
             self.send_queue_status()
 
+    def _first_movable_index(self, queue_name: str) -> int:
+        """
+        Index of the first queue item that may be moved: 1 while the active item is at the head,
+        paused or not, else 0. Nothing may move in front of the active item: the worker pops the
+        head once the active item is done, so anything placed before it would leave the finished
+        item in the queue to be started again.
+        """
+        scan_queue = self.queues[queue_name]
+        active = scan_queue.active_instruction_queue
+        if active is not None and scan_queue.queue and scan_queue.queue[0] is active:
+            return 1
+        return 0
+
     def _get_queue_item_by_scan_id(
         self, msg: messages.ScanQueueOrderMessage
     ) -> DirectInstructionQueueItem | None:

{bec_server-4.4.2 → bec_server-4.5.0}/bec_server/scan_server/scans/scan_actions.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 import functools
 import os
+import time
 import uuid
-from collections.abc import Iterable
+from collections.abc import Iterable, Iterator
+from contextlib import contextmanager
 from string import Template
 from typing import TYPE_CHECKING, Any, Callable, Literal, TypeAlias
 
@@ -18,6 +20,8 @@ from bec_lib.logger import bec_logger
 from bec_lib.messaging_hooks import MessagingEvent
 from bec_lib.messaging_services import NotificationMessageObject
 from bec_lib.utils.scan_utils import compose_cli_input_from_scan_info
+from bec_server.scan_server.errors import UserScanInterruption
+from bec_server.scan_server.scan_hook_broadcaster import ScanHookBroadcaster
 from bec_server.scan_server.scans.scan_status import ScanStatus
 
 if TYPE_CHECKING:
@@ -67,6 +71,84 @@ class ScanActions:
         self._metadata_suffix = ""
         self._queued_device_locks: set[str] = set()
         self._scan_running = False
+        self._scan_hook_stack: list[str] = []
+        self._step_started_at: float | None = None
+        self._hook_broadcaster = ScanHookBroadcaster(self._connector)
+
+    @contextmanager
+    def _scan_hook_context(self, hook: str) -> Iterator[None]:
+        """
+        Broadcast lifecycle hooks, reporting per-point execution as scan_core. The start, end and
+        failure of a top-level step are sent at once, so no step or error is lost to coalescing.
+        A step ended by a user or alarm interruption (finish now, abort, halt) is reported as
+        interrupted, any other exception as an error.
+        """
+        if hook == "at_each_point":
+            hook = "scan_core"
+        if self._scan_hook_stack and self._scan_hook_stack[-1] == hook:
+            yield
+            return
+        is_step = not self._scan_hook_stack
+        if is_step and self._hook_broadcaster.active:
+            self._step_started_at = time.time()
+        self._scan_hook_stack.append(hook)
+        try:
+            self._broadcast_scan_hook(flush=is_step)
+            yield
+        except UserScanInterruption:
+            self._broadcast_scan_hook(status="interrupted", flush=is_step)
+            raise
+        except BaseException:
+            self._broadcast_scan_hook(status="error", flush=is_step)
+            raise
+        else:
+            self._broadcast_scan_hook(status="completed", flush=is_step)
+        finally:
+            self._scan_hook_stack.pop()
+            if self._scan_hook_stack:
+                self._broadcast_scan_hook()
+            else:
+                self._step_started_at = None
+
+    def _broadcast_scan_hook(
+        self,
+        instructions: list[messages.DeviceInstructionMessage] | None = None,
+        status: Literal["running", "completed", "interrupted", "error"] = "running",
+        waiting: bool = False,
+        wait_started_at: float | None = None,
+        flush: bool = False,
+    ) -> None:
+        """
+        Publish the current hook with the scan's original correlation metadata.
+
+        Args:
+            flush (bool): Send at once instead of at the end of the 0.1 second window.
+        """
+        scan_info = self._scan.scan_info
+        if scan_info.scan_id is None:
+            return
+        self._hook_broadcaster.add(
+            messages.ScanHookMessage(
+                scan_id=scan_info.scan_id,
+                hook=self._scan_hook_stack[-1] if self._scan_hook_stack else None,
+                status=status,
+                instructions=instructions or [],
+                waiting=waiting,
+                wait_started_at=wait_started_at,
+                hook_started_at=self._step_started_at,
+                metadata={**scan_info.metadata, "scan_id": scan_info.scan_id},
+            )
+        )
+        if flush:
+            self._hook_broadcaster.flush_now()
+
+    def _on_status_wait(self, status: ScanStatus) -> None:
+        """Report pending instructions; the next lifecycle or instruction update replaces the wait."""
+        self._broadcast_scan_hook(
+            instructions=status.pending_instructions,
+            waiting=True,
+            wait_started_at=status.wait_started_at,
+        )
 
     @property
     def readout_priority(self) -> dict:
@@ -649,10 +731,13 @@ class ScanActions:
             metadata={"device_instr_id": status._device_instr_id},
         )
         self._send(instr)
-        if min_wait is not None:
-            self._shutdown_event.wait(min_wait)
-        if wait:
-            status.wait()
+        # The exposure delay blocks execution even if the trigger has already completed.
+        # pylint: disable=protected-access
+        with status._wait_context(waiting=bool(min_wait and min_wait > 0)):
+            if min_wait is not None:
+                self._shutdown_event.wait(min_wait)
+            if wait:
+                status.wait()
         return status
 
     @requires_scan_is_running
@@ -1045,6 +1130,7 @@ class ScanActions:
             registry=self._status_registry,
             is_container=is_container,
             name=name,
+            wait_callback=self._on_status_wait,
         )
         self._status_registry[status._device_instr_id] = status
         return status
@@ -1076,7 +1162,11 @@ class ScanActions:
         instr_devices = msg.device if isinstance(msg.device, list) else [msg.device]
         if set(instr_devices) & self._devices_with_required_response:
             msg.metadata["response"] = True
+        status = self._status_registry.get(msg.metadata.get("device_instr_id"))
+        if status is not None:
+            status.instruction = msg
         self._connector.send(MessageEndpoints.device_instructions(), msg)
+        self._broadcast_scan_hook(instructions=[msg])
 
     def _get_message_metadata(self) -> dict:
         metadata = {}

{bec_server-4.4.2 → bec_server-4.5.0}/bec_server/scan_server/scans/scan_modifier.py

@@ -101,8 +101,8 @@ def scan_hook(func):
     in the appropriate order.
     """
 
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
+    # pylint: disable=protected-access
+    def dispatch(self, *args, **kwargs):
         if func.__name__ not in self._scan_modifier_hooks:
             return func(self, *args, **kwargs)
 
@@ -131,6 +131,11 @@ def scan_hook(func):
 
         return
 
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        with self.actions._scan_hook_context(func.__name__):
+            return dispatch(self, *args, **kwargs)
+
     # pylint: disable=protected-access
     wrapper._scan_hook_info = {"method_name": func.__name__}  # type: ignore
     wrapper._scan_hook_original = func  # type: ignore[attr-defined]