bec-ipython-client 3.84.0__py3-none-any.whl → 3.89.3__py3-none-any.whl

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bec_ipython_client
-Version: 3.84.0
+Version: 3.89.3
 Summary: BEC IPython client
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

bec_ipython_client/callbacks/ipython_live_updates.py

@@ -31,7 +31,7 @@ class IPythonLiveUpdates:
         self._interrupted_request = None
         self._active_callback = None
         self._processed_instructions = 0
-        self._active_request = None
+        self._active_request: messages.ScanQueueMessage | None = None
         self._user_callback = None
         self._request_block_index = collections.defaultdict(lambda: 0)
         self._request_block_id = None
@@ -47,7 +47,9 @@ class IPythonLiveUpdates:
         Args:
             report_instructions (list): The list of report instructions.
         """
-        scan_type = self._active_request.content["scan_type"]
+        if not self._active_request:
+            return
+        scan_type = self._active_request.scan_type
         if scan_type in ["open_scan_def", "close_scan_def"]:
             self._process_instruction({"scan_progress": {"points": 0, "show_table": True}})
             return
@@ -74,6 +76,9 @@ class IPythonLiveUpdates:
         scan_report_type = list(instr.keys())[0]
         scan_def_id = self.client.scans._scan_def_id
         interactive_scan = self.client.scans._interactive_scan
+        if self._active_request is None:
+            # Already checked in caller method. It is just for type checking purposes.
+            return
         if scan_def_id is None or interactive_scan:
             if scan_report_type == "readback":
                 LiveUpdatesReadbackProgressbar(
@@ -126,17 +131,22 @@ class IPythonLiveUpdates:
             )
             self._active_callback.run()
 
-    def _available_req_blocks(self, queue: QueueItem, request: messages.ScanQueueMessage):
+    def _available_req_blocks(
+        self, queue: QueueItem, request: messages.ScanQueueMessage
+    ) -> list[messages.RequestBlock]:
         """Get the available request blocks.
 
         Args:
             queue (QueueItem): The queue item.
             request (messages.ScanQueueMessage): The request message.
+
+        Returns:
+            list[messages.RequestBlock]: The list of available request blocks.
         """
         available_blocks = [
             req_block
             for req_block in queue.request_blocks
-            if req_block["RID"] == request.metadata["RID"]
+            if req_block.RID == request.metadata["RID"]
         ]
         return available_blocks
 
@@ -167,7 +177,7 @@ class IPythonLiveUpdates:
 
                 available_blocks = self._available_req_blocks(queue, request)
                 req_block = available_blocks[self._request_block_index[req_id]]
-                report_instructions = req_block.get("report_instructions", [])
+                report_instructions = req_block.report_instructions or []
                 self._process_report_instructions(report_instructions)
 
             self._reset()
@@ -191,12 +201,19 @@ class IPythonLiveUpdates:
             self.client.queue.request_scan_halt()
 
     def _element_in_queue(self) -> bool:
-        queue = self.client.queue.queue_storage.current_scan_queue.get("primary", {}).get(
-            "info", []
-        )
-        if not queue:
+        if self.client.queue is None:
             return False
-        return self._current_queue.queue_id in queue[0].get("queue_id")
+        if (csq := self.client.queue.queue_storage.current_scan_queue) is None:
+            return False
+        scan_queue_status = csq.get(self.client.queue.get_default_scan_queue())
+        if scan_queue_status is None:
+            return False
+        queue_info = scan_queue_status.info
+        if not queue_info:
+            return False
+        if self._current_queue is None:
+            return False
+        return self._current_queue.queue_id == queue_info[0].queue_id
 
     def _process_queue(
         self, queue: QueueItem, request: messages.ScanQueueMessage, req_id: str
@@ -216,11 +233,15 @@ class IPythonLiveUpdates:
         if not queue.request_blocks or not queue.status or queue.queue_position is None:
             return False
         if queue.status == "PENDING" and queue.queue_position > 0:
-            status = self.client.queue.queue_storage.current_scan_queue.get("primary", {}).get(
-                "status"
+            target_queue = self.client.queue.queue_storage.current_scan_queue.get(
+                self.client.queue.get_default_scan_queue()
             )
+
+            if target_queue is None:
+                return False
+            status = target_queue.status
             print(
-                "Scan is enqueued and is waiting for execution. Current position in queue:"
+                f"Scan is enqueued and is waiting for execution. Current position in queue {self.client.queue.get_default_scan_queue()}:"
                 f" {queue.queue_position + 1}. Queue status: {status}.",
                 end="\r",
                 flush=True,
@@ -229,13 +250,13 @@ class IPythonLiveUpdates:
         if not available_blocks:
             return False
         req_block = available_blocks[self._request_block_index[req_id]]
-        if req_block["content"]["scan_type"] in [
+        if req_block.msg.scan_type in [
             "open_scan_def",
             "mv",
         ]:  # TODO: make this more general for all scan types that don't have report instructions
             return True
 
-        report_instructions = req_block["report_instructions"]
+        report_instructions = req_block.report_instructions or []
         if not report_instructions:
             return False
         self._process_report_instructions(report_instructions)
@@ -263,7 +284,11 @@ class IPythonLiveUpdates:
         self._current_queue = None
         self._user_callback = None
         self._processed_instructions = 0
-        scan_closed = forced or (self._active_request.content["scan_type"] == "close_scan_def")
+        scan_closed = (
+            forced
+            or self._active_request is None
+            or (self._active_request.scan_type == "close_scan_def")
+        )
         self._active_request = None
 
         if self.client.scans._scan_def_id and not scan_closed:

bec_ipython_client/callbacks/live_table.py

@@ -56,7 +56,6 @@ class LiveUpdatesTable(LiveUpdatesBase):
         super().__init__(
             bec, report_instruction=report_instruction, request=request, callbacks=callbacks
         )
-        self.scan_queue_request = None
         self.scan_item = None
         self.dev_values = None
         self.point_data = None
@@ -73,6 +72,8 @@ class LiveUpdatesTable(LiveUpdatesBase):
     def wait_for_scan_to_start(self):
         """wait until the scan starts"""
         while True:
+            if not self.scan_item or not self.scan_item.queue:
+                raise RuntimeError("No scan item or scan queue available.")
             queue_pos = self.scan_item.queue.queue_position
             self.check_alarms()
             if self.scan_item.status == "closed":
@@ -160,7 +161,20 @@ class LiveUpdatesTable(LiveUpdatesBase):
         return header
 
     def update_scan_item(self, timeout: float = 15):
-        """get the current scan item"""
+        """
+        Get the current scan item and update self.scan_item
+
+        Args:
+            timeout (float): timeout in seconds
+
+        Raises:
+            RuntimeError: if no scan queue request is available
+            TimeoutError: if no scan item is found before reaching the timeout
+
+        """
+        if not self.scan_queue_request:
+            raise RuntimeError("No scan queue request available.")
+
         start = time.time()
         while self.scan_queue_request.scan is None:
             self.check_alarms()
@@ -178,14 +192,19 @@ class LiveUpdatesTable(LiveUpdatesBase):
 
     def _wait_for_report_instructions(self):
         """wait until the report instructions are available"""
+        if not self.scan_queue_request or not self.scan_item or not self.scan_item.queue:
+            logger.warning(
+                f"Cannot wait for report instructions. scan_queue_request: {self.scan_queue_request}, scan_item: {self.scan_item}, scan_item.queue: {getattr(self.scan_item, 'queue', None)}"
+            )
+            return
         req_ID = self.scan_queue_request.requestID
         while True:
             request_block = [
-                req for req in self.scan_item.queue.request_blocks if req["RID"] == req_ID
+                req for req in self.scan_item.queue.request_blocks if req.RID == req_ID
             ][0]
-            if not request_block["is_scan"]:
+            if not request_block.is_scan:
                 break
-            if request_block["report_instructions"]:
+            if request_block.report_instructions:
                 break
             self.check_alarms()
 
@@ -195,6 +214,9 @@ class LiveUpdatesTable(LiveUpdatesBase):
         Args:
             target_num_points (int): number of points to be collected
         """
+        if not self.scan_item:
+            logger.warning("No scan item available for live updates.")
+            return
         with ScanProgressBar(
             scan_number=self.scan_item.scan_number, clear_on_exit=self._print_table_data
         ) as progressbar:

bec_ipython_client/callbacks/utils.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import abc
-import threading
 import time
 import traceback
 from collections.abc import Callable
@@ -21,23 +20,6 @@ class ScanRequestError(Exception):
     """Error raised when a scan request is rejected"""
 
 
-def set_event_delayed(event: threading.Event, delay: int) -> None:
-    """Set event with a delay
-
-    Args:
-        event (threading.Event): event that should be set
-        delay (int): delay time in seconds
-
-    """
-
-    def call_set():
-        time.sleep(delay)
-        event.set()
-
-    thread = threading.Thread(target=call_set, daemon=True)
-    thread.start()
-
-
 def check_alarms(bec):
     """check for alarms and raise them if needed"""
     bec.alarm_handler.raise_alarms()
@@ -62,7 +44,7 @@ class LiveUpdatesBase(abc.ABC):
         self.bec = bec
         self.request = request
         self.RID = request.metadata["RID"]
-        self.scan_queue_request = None
+        self.scan_queue_request: RequestItem | None = None
         self.report_instruction = report_instruction
         if callbacks is None:
             self.callbacks = []

bec_ipython_client/main.py

@@ -29,6 +29,7 @@ from bec_lib.bec_service import parse_cmdline_args
 from bec_lib.callback_handler import EventType
 from bec_lib.client import BECClient
 from bec_lib.logger import bec_logger
+from bec_lib.procedures.hli import ProcedureHli
 from bec_lib.redis_connector import RedisConnector
 from bec_lib.service_config import ServiceConfig
 from bec_lib.utils.pydantic_pretty_print import pretty_print_pydantic_validation_error
@@ -58,6 +59,7 @@ class BECIPythonClient:
     # the CLIBECClient but directly through the BECIPythonClient. While this is not
     # needed for normal usage, it is required, e.g. for mocks.
     _local_only_types: Tuple = ()
+    _client: CLIBECClient | BECClient
 
     def __init__(
         self,
@@ -75,6 +77,7 @@ class BECIPythonClient:
             name="BECIPythonClient",
             prompt_for_acl=True,
         )
+
         self._ip = IPython.get_ipython()
         self.started = False
         self._sighandler = None
@@ -150,6 +153,7 @@ class BECIPythonClient:
             object,
             lambda o, p, cycle: o.__str__ is object.__str__ and p.text(repr(o)) or p.text(str(o)),
         )
+        self._set_idle()
 
     def _update_namespace_callback(self, action: Literal["add", "remove"], ns_objects: dict):
         """Callback to update the global namespace of ipython.
@@ -187,13 +191,14 @@ class BECIPythonClient:
         magics = BECMagics(self._ip, self)
         self._ip.register_magics(magics)
 
-    def shutdown(self):
+    def shutdown(self, per_thread_timeout_s: float | None = None):
         """shutdown the client and all its components"""
         try:
             self.gui.close()
         except AttributeError:
             pass
-        self._client.shutdown()
+        self._client.shutdown(per_thread_timeout_s)
+        logger.success("done")
 
     def _create_exception_handler(self):
         return functools.partial(_ip_exception_handler, parent=self)
@@ -338,7 +343,6 @@ def main():
     parser = argparse.ArgumentParser(
         prog="BEC IPython client", description="BEC command line interface"
     )
-    parser.add_argument("--version", action="store_true", default=False)
     parser.add_argument("--nogui", action="store_true", default=False)
     parser.add_argument(
         "--gui-id",
@@ -358,10 +362,6 @@ def main():
     # remove already parsed args from command line args
     sys.argv = sys.argv[:1] + left_args
 
-    if args.version:
-        print(f"BEC IPython client: {version('bec_ipython_client')}")
-        sys.exit(0)
-
     if available_plugins and config.is_default():
         # check if config is defined in a plugin;
         # in this case the plugin config takes precedence over

bec_ipython_client/signals.py

@@ -1,9 +1,15 @@
+from __future__ import annotations
+
 import signal
 import threading
 import time
+from typing import TYPE_CHECKING
 
 from bec_lib.bec_errors import ScanInterruption
 
+if TYPE_CHECKING:  # pragma: no cover
+    from bec_lib.client import BECClient
+
 PAUSE_MSG = """
 The Scan Queue is entering a paused state. These are your options for changing
 the state of the queue:
@@ -73,7 +79,7 @@ class SignalHandler:
 
 
 class SigintHandler(SignalHandler):
-    def __init__(self, bec):
+    def __init__(self, bec: BECClient):
         super().__init__(signal.SIGINT)
         self.bec = bec
         self.last_sigint_time = None  # time most recent SIGINT was processed
@@ -84,11 +90,11 @@ class SigintHandler(SignalHandler):
         if not current_scan:
             raise KeyboardInterrupt
 
-        status = current_scan.get("status").lower()
+        status = current_scan.status.lower()
         if status not in ["running", "deferred_pause"]:
             raise KeyboardInterrupt
 
-        if any(current_scan.get("is_scan")) and (
+        if any(current_scan.is_scan) and (
             self.last_sigint_time is None or time.time() - self.last_sigint_time > 10
         ):
             # reset the counter to 1

{bec_ipython_client-3.84.0.dist-info → bec_ipython_client-3.89.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bec_ipython_client
-Version: 3.84.0
+Version: 3.89.3
 Summary: BEC IPython client
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

{bec_ipython_client-3.84.0.dist-info → bec_ipython_client-3.89.3.dist-info}/RECORD

@@ -1,21 +1,21 @@
 .gitignore,sha256=XxC6jyyftTo2CLtm4K8axuNPYwA9Wgaz2R93WhX8bTQ,3364
-PKG-INFO,sha256=ijJpv2TMYSqXFg607-vZwabQv7Ib-B-cWeiu71D9-Xc,1052
+PKG-INFO,sha256=trZwsE2OpNig8TZ6MjQ6HN5lX5tRxLrygSFHiz4FATs,1052
 demo.py,sha256=AquJB-0Tu2bIEFpcJ0Q3RUBt1xldqS6GgU5CjOAg_ww,7083
-pyproject.toml,sha256=HuXmixsLCNg82K4Z_GDHOdRWAVHza_oeQ7Pe5fKipHU,1229
+pyproject.toml,sha256=Gv8V_TLxvNXUR9W-ROwlP7dGq9cbC8BfRC9anNsoddQ,1229
 bec_ipython_client/__init__.py,sha256=ihd_V8I7Qo0MWKMo7bcvPf-ZyUQqkcNf8IAWLJKiFJE,79
 bec_ipython_client/beamline_mixin.py,sha256=scMWIFbHJajyECzbwEVKyQUGjpqA9C_KiU2M6FuRH_Q,1067
 bec_ipython_client/bec_magics.py,sha256=Rz2aXkUCeAV_VxdXqLUNHh8T44kSH9ha83OiEtdptzI,2792
 bec_ipython_client/bec_startup.py,sha256=GGlHyxnSCQfYF5n-pYq2ic0pSyW5zvnT2PAlI5kY77w,1930
-bec_ipython_client/main.py,sha256=XxSaiRuvDPKRzM3yuPz3FW5I0wnfyL2SgEfEG3VrDJg,13543
+bec_ipython_client/main.py,sha256=h7417qQj0rwj5gjbKg6pn8Y_yYJf4b8Fs5EP29sXXOg,13564
 bec_ipython_client/prettytable.py,sha256=TnhGPGuU0XEvnIYJ1UfTEwadcowFW4rqJW8z_Sm0EDw,2534
 bec_ipython_client/progressbar.py,sha256=aDKYjzXmGSwa82ewm59V8WSuqVQz9GiZPx5G65fEwpk,11090
-bec_ipython_client/signals.py,sha256=mbThPo6h3mQ6RFRm9viETDMC_unFa7QxiymCdM_ZK7U,4194
+bec_ipython_client/signals.py,sha256=P7FrIB66dGdPESUngBnIVPGrAhcd2Tek8ufq9ALBoYU,4340
 bec_ipython_client/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bec_ipython_client/callbacks/device_progress.py,sha256=NzWIO1oowxpN14hPXBne6RfZxOW82EhwDM3jHLnn3xI,2603
-bec_ipython_client/callbacks/ipython_live_updates.py,sha256=Cx62yJSzIcRvAA_lC86PSs8825hJtk22yBPhuJjgRPc,10520
-bec_ipython_client/callbacks/live_table.py,sha256=3qmwCEjqe-xFqo3Ez2dhKchawlr0cOmNgzpg857-gp0,13503
+bec_ipython_client/callbacks/ipython_live_updates.py,sha256=zaXR6UX7fA01-N0B0C2t67lOqjBgq8nal8eZpgf_nDc,11416
+bec_ipython_client/callbacks/live_table.py,sha256=ewdter1z1arV1EvEsReOihluyTmAHMrHtOu_S29vixo,14432
 bec_ipython_client/callbacks/move_device.py,sha256=TkbtKFBr1nuJ8THTSm70Y8D7_va2YQ57oBGGt0BavW4,8153
-bec_ipython_client/callbacks/utils.py,sha256=jS92ndZNU2t5n9FsLrSbulmUIeepHhwdQdfO60YjiJk,5702
+bec_ipython_client/callbacks/utils.py,sha256=e8USLnufNTpI2Bkp-sjRsSFvIXxB2CirF6WRPVsOets,5338
 bec_ipython_client/high_level_interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bec_ipython_client/high_level_interfaces/bec_hli.py,sha256=G1eF-lbLEDuN2mr_AYSxhhGb4LT6P72g0a4sU9M9zgk,1221
 bec_ipython_client/high_level_interfaces/spec_hli.py,sha256=z7WtjiC4LtMfKJn12SbguHPCLqbAsZNfUDyiUW0LOiU,5818
@@ -30,15 +30,15 @@ tests/client_tests/conftest.py,sha256=dUvYqvl6wogxN-kRCUuo_Uzna3P2uSYD7qGHyMGBlP
 tests/client_tests/test_beamline_mixins.py,sha256=8Ws0bmzl2gSW0VuOVu80_JbYNb5Y-htchmrrptwjwDo,4611
 tests/client_tests/test_bec_client.py,sha256=gks7IYoDDXpbkC_EGWQfKtH08Yig71PRZtleXFhkU8A,8903
 tests/client_tests/test_device_progress.py,sha256=GEw2g8MQZnv5mxABEZlxBmaMpxVS33wogaYohFolDEs,2353
-tests/client_tests/test_ipython_live_updates.py,sha256=HpA16Mx0WdseqlfP0FojlAd6VfvjnwAUReDkItZVYWs,6081
+tests/client_tests/test_ipython_live_updates.py,sha256=s4HLqF2hVQaaJhRKv0dxj0DiZK5M9Z_WwMAbGk19EFg,11918
 tests/client_tests/test_live_table.py,sha256=0EKgWOgDqpjN8fJEeAzoKNCwFneDEsUY2NfWQkxB6xc,18155
 tests/client_tests/test_move_callback.py,sha256=bUCcWoz_tc-yRAtwmEUMrKE_qKwatop_wReSugGGEIQ,8670
 tests/client_tests/test_pretty_table.py,sha256=uQ-KPb3RXoCFE_t1IrpkT6kZAoqW7pFXxbFc445sX0Y,469
-tests/end-2-end/_ensure_requirements_container.py,sha256=RT2a5cUZnXtMjJRlPImGzgyX-KJPLiav5WSp_l3wzv8,725
-tests/end-2-end/test_procedures_e2e.py,sha256=xjczbB33Cf5pDLe1KWcalRRBrSxOQ3cbV3s3w3GsC_A,4922
-tests/end-2-end/test_scans_e2e.py,sha256=GMStotxqu24794TxIFc55PCtOjuG2qbkmibcUj7r-DI,31069
+tests/end-2-end/_ensure_requirements_container.py,sha256=dv_ACfo9nHyiiEL3AMccaEg38VxGqB0W7u_iPyCVeiE,701
+tests/end-2-end/test_procedures_e2e.py,sha256=ai8pUrHaL-5sYuaKYucuYEwyvhelS1dsW9KGoBb1Riw,4872
+tests/end-2-end/test_scans_e2e.py,sha256=JhyWjDyUlBZAM_gL6QtfKXfI_fXdUnykYCIWCSorRiA,32527
 tests/end-2-end/test_scans_lib_e2e.py,sha256=dqs0ojkyQWStIQbqABq9mQrjqQyE43gr37VPhxvQ8b8,19427
-bec_ipython_client-3.84.0.dist-info/METADATA,sha256=ijJpv2TMYSqXFg607-vZwabQv7Ib-B-cWeiu71D9-Xc,1052
-bec_ipython_client-3.84.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-bec_ipython_client-3.84.0.dist-info/entry_points.txt,sha256=oQUXYY0jjD9ZvKPHwaGn2wkUIWpDZM8L4ixDA3RlBWE,53
-bec_ipython_client-3.84.0.dist-info/RECORD,,
+bec_ipython_client-3.89.3.dist-info/METADATA,sha256=trZwsE2OpNig8TZ6MjQ6HN5lX5tRxLrygSFHiz4FATs,1052
+bec_ipython_client-3.89.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+bec_ipython_client-3.89.3.dist-info/entry_points.txt,sha256=oQUXYY0jjD9ZvKPHwaGn2wkUIWpDZM8L4ixDA3RlBWE,53
+bec_ipython_client-3.89.3.dist-info/RECORD,,

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "bec_ipython_client"
-version = "3.84.0"
+version = "3.89.3"
 description = "BEC IPython client"
 requires-python = ">=3.11"
 classifiers = [

tests/client_tests/test_ipython_live_updates.py

@@ -7,34 +7,94 @@ from bec_lib import messages
 from bec_lib.queue_items import QueueItem
 
 
-@pytest.mark.timeout(20)
-def test_live_updates_process_queue_pending(bec_client_mock):
+@pytest.fixture
+def queue_elements(bec_client_mock):
     client = bec_client_mock
-    live_updates = IPythonLiveUpdates(client)
     request_msg = messages.ScanQueueMessage(
         scan_type="grid_scan",
         parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
         queue="primary",
         metadata={"RID": "something"},
     )
+    request_block = messages.RequestBlock(
+        msg=request_msg,
+        RID="req_id",
+        scan_motors=["samx"],
+        report_instructions=[],
+        readout_priority={"monitored": ["samx"]},
+        is_scan=True,
+        scan_number=1,
+        scan_id="scan_id",
+    )
     queue = QueueItem(
         scan_manager=client.queue,
         queue_id="queue_id",
-        request_blocks=[request_msg],
+        request_blocks=[request_block],
         status="PENDING",
         active_request_block={},
         scan_id=["scan_id"],
     )
-    client.queue.queue_storage.current_scan_queue = {"primary": {"status": "RUNNING"}}
+    return queue, request_block, request_msg
+
+
+@pytest.fixture
+def sample_request_msg():
+    return messages.ScanQueueMessage(
+        scan_type="grid_scan",
+        parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
+        queue="primary",
+        metadata={"RID": "something"},
+    )
+
+
+@pytest.fixture
+def sample_request_block(sample_request_msg):
+    return messages.RequestBlock(
+        msg=sample_request_msg,
+        RID="req_id",
+        scan_motors=["samx"],
+        report_instructions=[],
+        readout_priority={"monitored": ["samx"]},
+        is_scan=True,
+        scan_number=1,
+        scan_id="scan_id",
+    )
+
+
+@pytest.fixture
+def sample_queue_info_entry(sample_request_block):
+    return messages.QueueInfoEntry(
+        queue_id="test_queue_id",
+        scan_id=["scan_id"],
+        is_scan=[True],
+        request_blocks=[sample_request_block],
+        scan_number=[1],
+        status="RUNNING",
+        active_request_block=None,
+    )
+
+
+@pytest.fixture
+def sample_scan_queue_status(sample_queue_info_entry):
+    return messages.ScanQueueStatus(info=[sample_queue_info_entry], status="RUNNING")
+
+
+@pytest.mark.timeout(20)
+def test_live_updates_process_queue_pending(bec_client_mock, queue_elements):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+    queue, request_block, request_msg = queue_elements
+
+    client.queue.queue_storage.current_scan_queue = {
+        "primary": messages.ScanQueueStatus(info=[], status="RUNNING")
+    }
     with mock.patch.object(queue, "_update_with_buffer"):
         with mock.patch(
             "bec_lib.queue_items.QueueItem.queue_position", new_callable=mock.PropertyMock
         ) as queue_pos:
             queue_pos.return_value = 2
             with mock.patch.object(
-                live_updates,
-                "_available_req_blocks",
-                return_value=[{"report_instructions": [], "content": {"scan_type": "grid_scan"}}],
+                live_updates, "_available_req_blocks", return_value=[request_block]
             ):
                 with mock.patch.object(live_updates, "_process_report_instructions") as process:
                     with mock.patch("builtins.print") as prt:
@@ -45,39 +105,30 @@ def test_live_updates_process_queue_pending(bec_client_mock):
 
 
 @pytest.mark.timeout(20)
-def test_live_updates_process_queue_running(bec_client_mock):
+def test_live_updates_process_queue_running(bec_client_mock, queue_elements):
     client = bec_client_mock
     live_updates = IPythonLiveUpdates(client)
-    request_msg = messages.ScanQueueMessage(
-        scan_type="grid_scan",
-        parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
-        queue="primary",
-        metadata={"RID": "something"},
-    )
+    _, request_block, request_msg = queue_elements
     queue = QueueItem(
         scan_manager=client.queue,
         queue_id="queue_id",
-        request_blocks=[request_msg],
+        request_blocks=[request_block],
         status="RUNNING",
         active_request_block={},
         scan_id=["scan_id"],
     )
     live_updates._active_request = request_msg
-    client.queue.queue_storage.current_scan_queue = {"primary": {"status": "RUNNING"}}
+    request_block.report_instructions = [{"wait_table": 10}]
+    client.queue.queue_storage.current_scan_queue = {
+        "primary": messages.ScanQueueStatus(info=[], status="RUNNING")
+    }
     with mock.patch.object(queue, "_update_with_buffer"):
         with mock.patch(
             "bec_lib.queue_items.QueueItem.queue_position", new_callable=mock.PropertyMock
         ) as queue_pos:
             queue_pos.return_value = 2
             with mock.patch.object(
-                live_updates,
-                "_available_req_blocks",
-                return_value=[
-                    {
-                        "report_instructions": [{"wait_table": 10}],
-                        "content": {"scan_type": "grid_scan"},
-                    }
-                ],
+                live_updates, "_available_req_blocks", return_value=[request_block]
             ):
                 with mock.patch.object(live_updates, "_process_instruction") as process:
                     with mock.patch("builtins.print") as prt:
@@ -88,37 +139,19 @@ def test_live_updates_process_queue_running(bec_client_mock):
 
 
 @pytest.mark.timeout(20)
-def test_live_updates_process_queue_without_status(bec_client_mock):
+def test_live_updates_process_queue_without_status(bec_client_mock, queue_elements):
     client = bec_client_mock
     live_updates = IPythonLiveUpdates(client)
-    request_msg = messages.ScanQueueMessage(
-        scan_type="grid_scan",
-        parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
-        queue="primary",
-        metadata={"RID": "something"},
-    )
-    queue = QueueItem(
-        scan_manager=client.queue,
-        queue_id="queue_id",
-        request_blocks=[request_msg],
-        status=None,
-        active_request_block={},
-        scan_id=["scan_id"],
-    )
+    queue, _, request_msg = queue_elements
     with mock.patch.object(queue, "_update_with_buffer"):
         assert live_updates._process_queue(queue, request_msg, "req_id") is False
 
 
 @pytest.mark.timeout(20)
-def test_live_updates_process_queue_without_queue_number(bec_client_mock):
+def test_live_updates_process_queue_without_queue_number(bec_client_mock, queue_elements):
     client = bec_client_mock
     live_updates = IPythonLiveUpdates(client)
-    request_msg = messages.ScanQueueMessage(
-        scan_type="grid_scan",
-        parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
-        queue="primary",
-        metadata={"RID": "something"},
-    )
+    queue, _, request_msg = queue_elements
 
     with mock.patch(
         "bec_lib.queue_items.QueueItem.queue_position", new_callable=mock.PropertyMock
@@ -136,24 +169,182 @@ def test_live_updates_process_queue_without_queue_number(bec_client_mock):
             assert live_updates._process_queue(queue, request_msg, "req_id") is False
 
 
-# @pytest.mark.timeout(20)
-# @pytest.mark.asyncio
-# def test_live_updates_process_instruction_readback(bec_client_mock):
-#     client = bec_client_mock
-#     live_updates = IPythonLiveUpdates(client)
-#     request_msg = messages.ScanQueueMessage(
-#         scan_type="grid_scan",
-#         parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
-#         queue="primary",
-#         metadata={"RID": "something"},
-#     )
-#     live_updates._active_request = request_msg
-#     live_updates._user_callback = []
-#     client.queue.queue_storage.current_scan_queue = {"primary": {"status": "RUNNING"}}
-#     with mock.patch(
-#         "bec_client_mock.callbacks.ipython_live_updates.LiveUpdatesTable", new_callable=mock.Co
-#     ) as table:
-#         live_updates._process_instruction({"scan_progress": 10})
-#         table.assert_called_once_with(
-#             client, report_instructions={"scan_progress": 10}, request=request_msg, callbacks=[]
-#         )
+@pytest.mark.timeout(20)
+def test_available_req_blocks(bec_client_mock, queue_elements):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+    queue, request_block, request_msg = queue_elements
+
+    # Test with matching RID
+    available_blocks = live_updates._available_req_blocks(queue, request_msg)
+    assert (
+        len(available_blocks) == 0
+    )  # request_block.RID is "req_id", request_msg.metadata["RID"] is "something"
+
+    # Test with correct RID
+    request_block.RID = "something"
+    available_blocks = live_updates._available_req_blocks(queue, request_msg)
+    assert len(available_blocks) == 1
+    assert available_blocks[0] == request_block
+
+
+@pytest.mark.timeout(20)
+def test_available_req_blocks_multiple_blocks(bec_client_mock):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    request_msg = messages.ScanQueueMessage(
+        scan_type="grid_scan",
+        parameter={"args": {"samx": (-5, 5, 3)}, "kwargs": {}},
+        queue="primary",
+        metadata={"RID": "test_rid"},
+    )
+
+    request_block1 = messages.RequestBlock(
+        msg=request_msg,
+        RID="test_rid",
+        scan_motors=["samx"],
+        report_instructions=[],
+        readout_priority={"monitored": ["samx"]},
+        is_scan=True,
+        scan_number=1,
+        scan_id="scan_id_1",
+    )
+
+    request_block2 = messages.RequestBlock(
+        msg=request_msg,
+        RID="test_rid",
+        scan_motors=["samy"],
+        report_instructions=[],
+        readout_priority={"monitored": ["samy"]},
+        is_scan=True,
+        scan_number=2,
+        scan_id="scan_id_2",
+    )
+
+    request_block3 = messages.RequestBlock(
+        msg=request_msg,
+        RID="different_rid",
+        scan_motors=["samz"],
+        report_instructions=[],
+        readout_priority={"monitored": ["samz"]},
+        is_scan=True,
+        scan_number=3,
+        scan_id="scan_id_3",
+    )
+
+    queue = QueueItem(
+        scan_manager=client.queue,
+        queue_id="queue_id",
+        request_blocks=[request_block1, request_block2, request_block3],
+        status="RUNNING",
+        active_request_block={},
+        scan_id=["scan_id_1", "scan_id_2", "scan_id_3"],
+    )
+
+    available_blocks = live_updates._available_req_blocks(queue, request_msg)
+    assert len(available_blocks) == 2
+    assert request_block1 in available_blocks
+    assert request_block2 in available_blocks
+    assert request_block3 not in available_blocks
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_no_queue(bec_client_mock):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when client.queue is None
+    client.queue = None
+    assert live_updates._element_in_queue() is False
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_no_current_scan_queue(bec_client_mock):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when current_scan_queue is None
+    client.queue.queue_storage.current_scan_queue = None
+    assert live_updates._element_in_queue() is False
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_no_primary_queue(bec_client_mock):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when primary queue doesn't exist
+    scan_queue_status = messages.ScanQueueStatus(info=[], status="RUNNING")
+    client.queue.queue_storage.current_scan_queue = {"secondary": scan_queue_status}
+    assert live_updates._element_in_queue() is False
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_no_queue_info(bec_client_mock):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when queue_info is empty
+    scan_queue_status = messages.ScanQueueStatus(info=[], status="RUNNING")
+    client.queue.queue_storage.current_scan_queue = {"primary": scan_queue_status}
+    assert live_updates._element_in_queue() is False
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_no_current_queue(bec_client_mock, sample_scan_queue_status):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when _current_queue is None
+    live_updates._current_queue = None
+    client.queue.queue_storage.current_scan_queue = {"primary": sample_scan_queue_status}
+    assert live_updates._element_in_queue() is False
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_queue_id_not_in_info(bec_client_mock, sample_request_block):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when queue_id is not in info
+    current_queue = mock.MagicMock()
+    current_queue.queue_id = "my_queue_id"
+    live_updates._current_queue = current_queue
+
+    queue_info_entry = messages.QueueInfoEntry(
+        queue_id="different_queue_id",
+        scan_id=["scan_id"],
+        is_scan=[True],
+        request_blocks=[sample_request_block],
+        scan_number=[1],
+        status="RUNNING",
+        active_request_block=None,
+    )
+    scan_queue_status = messages.ScanQueueStatus(info=[queue_info_entry], status="RUNNING")
+    client.queue.queue_storage.current_scan_queue = {"primary": scan_queue_status}
+    assert live_updates._element_in_queue() is False
+
+
+@pytest.mark.timeout(20)
+def test_element_in_queue_queue_id_in_info(bec_client_mock, sample_request_block):
+    client = bec_client_mock
+    live_updates = IPythonLiveUpdates(client)
+
+    # Test when queue_id is in info (should return True)
+    current_queue = mock.MagicMock()
+    current_queue.queue_id = "my_queue_id"
+    live_updates._current_queue = current_queue
+
+    queue_info_entry = messages.QueueInfoEntry(
+        queue_id="my_queue_id",
+        scan_id=["scan_id"],
+        is_scan=[True],
+        request_blocks=[sample_request_block],
+        scan_number=[1],
+        status="RUNNING",
+        active_request_block=None,
+    )
+    scan_queue_status = messages.ScanQueueStatus(info=[queue_info_entry], status="RUNNING")
+    client.queue.queue_storage.current_scan_queue = {"primary": scan_queue_status}
+    assert live_updates._element_in_queue() is True

tests/end-2-end/_ensure_requirements_container.py

@@ -1,7 +1,7 @@
 from time import sleep
 
-from bec_server.scan_server.procedures.constants import PROCEDURE, ProcedureWorkerError
-from bec_server.scan_server.procedures.container_utils import PodmanCliUtils
+from bec_server.procedures.constants import PROCEDURE, ProcedureWorkerError
+from bec_server.procedures.container_utils import PodmanCliUtils
 
 image_name = (
     f"ghcr.io/bec-project/{PROCEDURE.CONTAINER.REQUIREMENTS_IMAGE_NAME}:v{PROCEDURE.BEC_VERSION}"

tests/end-2-end/test_procedures_e2e.py

@@ -12,10 +12,10 @@ from bec_ipython_client.main import BECIPythonClient
 from bec_lib import messages
 from bec_lib.endpoints import MessageEndpoints
 from bec_lib.logger import bec_logger
-from bec_server.scan_server.procedures.constants import _CONTAINER, _WORKER
-from bec_server.scan_server.procedures.container_utils import get_backend
-from bec_server.scan_server.procedures.container_worker import ContainerProcedureWorker
-from bec_server.scan_server.procedures.manager import ProcedureManager
+from bec_server.procedures.constants import _CONTAINER, _WORKER
+from bec_server.procedures.container_utils import get_backend
+from bec_server.procedures.container_worker import ContainerProcedureWorker
+from bec_server.procedures.manager import ProcedureManager
 
 if TYPE_CHECKING:
     from pytest_bec_e2e.plugin import LogTestTool
@@ -43,9 +43,9 @@ def client_logtool_and_manager(
     bec_ipython_client_fixture_with_logtool: tuple[BECIPythonClient, "LogTestTool"],
 ) -> Generator[tuple[BECIPythonClient, "LogTestTool", ProcedureManager], None, None]:
     client, logtool = bec_ipython_client_fixture_with_logtool
-    server = MagicMock()
-    server.bootstrap_server = f"{client.connector.host}:{client.connector.port}"
-    manager = ProcedureManager(server, ContainerProcedureWorker)
+    manager = ProcedureManager(
+        f"{client.connector.host}:{client.connector.port}", ContainerProcedureWorker
+    )
     yield client, logtool, manager
     manager.shutdown()
 
@@ -67,7 +67,8 @@ def test_building_worker_image():
 
 
 @pytest.mark.timeout(100)
-@patch("bec_server.scan_server.procedures.manager.procedure_registry.is_registered", lambda _: True)
+@patch("bec_server.procedures.manager.procedure_registry.is_registered", lambda _: True)
+@patch("bec_server.procedures.container_worker.PROCEDURE", PATCHED_CONSTANTS())
 def test_procedure_runner_spawns_worker(
     client_logtool_and_manager: tuple[BECIPythonClient, "LogTestTool", ProcedureManager],
 ):
@@ -75,7 +76,7 @@ def test_procedure_runner_spawns_worker(
     assert manager._active_workers == {}
     endpoint = MessageEndpoints.procedure_request()
     msg = messages.ProcedureRequestMessage(
-        identifier="sleep", args_kwargs=((), {"time_s": 2}), queue="test"
+        identifier="sleep", args_kwargs=((), {"time_s": 0.1}), queue="test"
     )
 
     logs = []
@@ -88,14 +89,14 @@ def test_procedure_runner_spawns_worker(
     client.connector.xadd(topic=endpoint, msg_dict=msg.model_dump())
 
     _wait_while(lambda: manager._active_workers == {}, 5)
-    _wait_while(lambda: manager._active_workers != {}, 20)
+    _wait_while(lambda: manager._active_workers != {}, 90)
 
     assert logs != []
 
 
 @pytest.mark.timeout(100)
-@patch("bec_server.scan_server.procedures.manager.procedure_registry.is_registered", lambda _: True)
-@patch("bec_server.scan_server.procedures.container_worker.PROCEDURE", PATCHED_CONSTANTS())
+@patch("bec_server.procedures.manager.procedure_registry.is_registered", lambda _: True)
+@patch("bec_server.procedures.container_worker.PROCEDURE", PATCHED_CONSTANTS())
 def test_happy_path_container_procedure_runner(
     client_logtool_and_manager: tuple[BECIPythonClient, "LogTestTool", ProcedureManager],
 ):
@@ -111,7 +112,7 @@ def test_happy_path_container_procedure_runner(
     conn.xadd(topic=endpoint, msg_dict=msg.model_dump())
 
     _wait_while(lambda: manager._active_workers == {}, 5)
-    _wait_while(lambda: manager._active_workers != {}, 20)
+    _wait_while(lambda: manager._active_workers != {}, 90)
 
     logtool.fetch()
     assert logtool.is_present_in_any_message("procedure accepted: True, message:")

tests/end-2-end/test_scans_e2e.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import _thread
 import io
 import os
 import threading
 import time
 from contextlib import redirect_stdout
+from typing import TYPE_CHECKING
 from unittest.mock import PropertyMock
 
 import h5py
@@ -19,6 +22,9 @@ from bec_lib.logger import bec_logger
 
 logger = bec_logger.logger
 
+if TYPE_CHECKING:  # pragma: no cover
+    from bec_ipython_client.main import BECIPythonClient
+
 # pylint: disable=protected-access
 
 
@@ -201,13 +207,13 @@ def test_mv_scan_mv(bec_ipython_client_fixture):
 
 
 @pytest.mark.timeout(100)
-def test_scan_abort(bec_ipython_client_fixture):
+def test_scan_abort(bec_ipython_client_fixture: BECIPythonClient):
     def send_abort(bec):
         while True:
             current_scan_info = bec.queue.scan_storage.current_scan_info
             if not current_scan_info:
                 continue
-            status = current_scan_info.get("status").lower()
+            status = current_scan_info.status.lower()
             if status not in ["running", "deferred_pause"]:
                 continue
             if bec.queue.scan_storage.current_scan is None:
@@ -217,7 +223,7 @@ def test_scan_abort(bec_ipython_client_fixture):
                 break
         while True:
             queue = bec.queue.queue_storage.current_scan_queue
-            if queue["primary"]["info"][0]["status"] == "DEFERRED_PAUSE":
+            if queue["primary"].info[0].status == "DEFERRED_PAUSE":
                 break
             time.sleep(0.5)
         _thread.interrupt_main()
@@ -241,7 +247,7 @@ def test_scan_abort(bec_ipython_client_fixture):
         time.sleep(0.5)
 
     current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
-    while current_queue["info"] or current_queue["status"] != "RUNNING":
+    while current_queue.info or current_queue.status != "RUNNING":
         time.sleep(0.5)
         current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
 
@@ -307,7 +313,7 @@ def test_queued_scan(bec_ipython_client_fixture):
     while len(scan2.scan.live_data) != 50:
         time.sleep(0.5)
     current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
-    while current_queue["info"] or current_queue["status"] != "RUNNING":
+    while current_queue.info or current_queue.status != "RUNNING":
         time.sleep(0.5)
         current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
     scan_number_end = bec.queue.next_scan_number
@@ -356,7 +362,7 @@ def test_scan_restart(bec_ipython_client_fixture):
     scan2.wait()
 
     current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
-    while current_queue["info"] or current_queue["status"] != "RUNNING":
+    while current_queue.info or current_queue.status != "RUNNING":
         time.sleep(0.5)
         current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
     scan_number_end = bec.queue.next_scan_number
@@ -364,7 +370,7 @@ def test_scan_restart(bec_ipython_client_fixture):
 
 
 @pytest.mark.timeout(100)
-def test_scan_observer_repeat_queued(bec_ipython_client_fixture):
+def test_scan_observer_repeat_queued(bec_ipython_client_fixture: BECIPythonClient):
     bec = bec_ipython_client_fixture
     bec.metadata.update({"unit_test": "test_scan_observer_repeat_queued"})
     scans = bec.scans
@@ -396,7 +402,7 @@ def test_scan_observer_repeat_queued(bec_ipython_client_fixture):
     scan2.wait()
 
     current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
-    while current_queue["info"] or current_queue["status"] != "RUNNING":
+    while current_queue.info or current_queue.status != "RUNNING":
         time.sleep(0.5)
         current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
     scan_number_end = bec.queue.next_scan_number
@@ -433,7 +439,7 @@ def test_scan_observer_repeat(bec_ipython_client_fixture):
         scan1.wait()
 
     current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
-    while current_queue["info"] or current_queue["status"] != "RUNNING":
+    while current_queue.info or current_queue.status != "RUNNING":
         time.sleep(0.5)
         current_queue = bec.queue.queue_storage.current_scan_queue["primary"]
     while True:
@@ -819,3 +825,46 @@ def test_client_info_message(bec_ipython_client_fixture):
         s1 = scans.line_scan(dev.samx, 0, 1, steps=10, exp_time=0.5, relative=False)
         output = buffer.getvalue()
         assert "test_client_info_message" in output
+
+
+@pytest.mark.timeout(100)
+def test_device_progress_grid_scan(bec_ipython_client_fixture, capsys):
+    bec = bec_ipython_client_fixture
+    scans = bec.scans
+    bec.metadata.update({"unit_test": "test_device_progress_grid_scan"})
+    dev = bec.device_manager.devices
+    scans.device_progress_grid_scan(
+        dev.samx, -5, 5, 10, dev.samy, -5, 5, 10, relative=True, exp_time=0.01
+    )
+    captured = capsys.readouterr()
+    assert "bpm4i" not in captured.out
+    assert "samx" not in captured.out
+    assert "Scan" in captured.out
+    assert "100 %" in captured.out
+
+
+@pytest.mark.timeout(100)
+def test_grid_scan_secondary_queue(capsys, bec_ipython_client_fixture):
+    bec = bec_ipython_client_fixture
+    scans = bec.scans
+    bec.metadata.update({"unit_test": "test_grid_scan_secondary_queue"})
+    dev = bec.device_manager.devices
+    status = scans.grid_scan(
+        dev.samx,
+        -5,
+        5,
+        10,
+        dev.samy,
+        -5,
+        5,
+        10,
+        exp_time=0.01,
+        relative=False,
+        scan_queue="secondary",
+    )
+    assert len(status.scan.live_data) == 100
+    assert status.scan.num_points == 100
+    captured = capsys.readouterr()
+    assert "finished. Scan ID" in captured.out
+
+    assert "secondary" in bec.queue.queue_storage.current_scan_queue