bec-server 4.3.0__tar.gz → 4.3.1__tar.gz

{bec_server-4.3.0 → bec_server-4.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: bec-server
-Version: 4.3.0
+Version: 4.3.1
 Summary: BEC server
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

{bec_server-4.3.0 → bec_server-4.3.1}/bec_server/actors/worker.py

@@ -1,14 +1,12 @@
 import importlib
 import os
-from contextlib import redirect_stdout
 from inspect import isclass
 
 from bec_lib.client import BECClient
-from bec_lib.logger import LogLevel, bec_logger
+from bec_lib.logger import bec_logger
 from bec_lib.messages import ProcedureWorkerStatus
-from bec_lib.redis_connector import RedisConnector
 from bec_server.actors.actor import ActorBase
-from bec_server.procedures.oop_worker_base import RedisOutputDiverter, get_env, push_status, setup
+from bec_server.procedures.oop_worker_base import get_env, push_status, setup
 from bec_server.procedures.subprocess_worker import SubProcessWorker
 
 logger = bec_logger.logger
@@ -53,20 +51,12 @@ def get_actor_env():
 def main():
     env, helper, client, conn = setup(get_env())
     actor_env = get_actor_env()
-    logger_connector = RedisConnector(env["redis_server"])
-    output_diverter = RedisOutputDiverter(logger_connector, env["queue"])
-    with redirect_stdout(output_diverter):
-        logger.add(
-            output_diverter,
-            level=LogLevel.SUCCESS,
-            format=bec_logger.formatting(is_container=True),
-            filter=bec_logger.filter(),
-        )
-        logger.success(f"Starting ActorProcedureWorker with env: {actor_env}")
+    # The parent SubProcessWorker streams stdout and stderr to Redis.
+    logger.success(f"Starting ActorProcedureWorker with env: {actor_env}")
     push_status(conn, env["queue"], ProcedureWorkerStatus.IDLE)
     actor_procedure(bec=client, **actor_env)
+    push_status(conn, env["queue"], ProcedureWorkerStatus.FINISHED)
     conn.shutdown()
-    logger_connector.shutdown()
 
 
 if __name__ == "__main__":

{bec_server-4.3.0 → bec_server-4.3.1}/bec_server/procedures/container_utils.py

@@ -220,7 +220,7 @@ class PodmanCliUtils(_PodmanUtilsBase):
         _name_arg = ["--replace", "--name", container_name] if container_name else []
         if container_name:
             # in case of incomplete cleanup previously, even with --replace it is possible to get an error
-            _run_and_capture_error("podman", "rm", "-f", container_name)
+            _run_and_capture_error("podman", "rm", "-f", "--ignore", container_name)
         return (
             _run_and_capture_error(
                 "podman",
@@ -251,7 +251,8 @@ class PodmanCliUtils(_PodmanUtilsBase):
 
     def logs(self, id: str) -> list[str]:
         try:
-            return _run_and_capture_error("podman", "logs", id).stderr.decode().splitlines()
+            output = _run_and_capture_error("podman", "logs", id)
+            return (output.stdout + output.stderr).decode(errors="replace").splitlines()
         except ProcedureWorkerError as e:
             logger.error(e)
             return [f"No logs found for container {id}\n"]

{bec_server-4.3.0 → bec_server-4.3.1}/bec_server/procedures/container_worker.py

@@ -28,7 +28,7 @@ class ContainerProcedureWorker(OutOfProcessWorkerBase):
             self._backend.build_worker_image()
         self._container_id = self._backend.run(
             image_tag,
-            self._worker_environment(),
+            self._worker_environment() | self._additional_env,
             [
                 {
                     "source": str(PROCEDURE.CONTAINER.DEPLOYMENT_PATH),

{bec_server-4.3.0 → bec_server-4.3.1}/bec_server/procedures/manager.py

@@ -46,6 +46,8 @@ class RedisCredentials(TypedDict):
 
 def _log_on_end(future: Future):
     """Use as a callback so that future is always done."""
+    if future.cancelled():
+        return
     if e := future.exception():
         logger.error(f"Worker failed with exception: {e}")
     else:
@@ -141,9 +143,9 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         with self.lock:
             if exec_msg.queue not in self._active_workers:
                 new_worker = self.executor.submit(self.spawn, queue=queue, env=env)
+                self._active_workers[queue] = {"worker": None, "future": new_worker}
                 new_worker.add_done_callback(_log_on_end)
                 new_worker.add_done_callback(self._cleanup_worker_function(queue))
-                self._active_workers[queue] = {"worker": None, "future": new_worker}
         return exec_msg
 
     def spawn(self, queue: str, env: dict[str, str]):
@@ -156,11 +158,20 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
             raise WorkerAlreadyExists(
                 f"Queue {queue} already has an active worker in {self._active_workers}!"
             )
-        with self._worker_cls(self._server, queue, PROCEDURE.WORKER.QUEUE_TIMEOUT_S, env) as worker:
-            with self.lock:
-                self._active_workers[queue]["worker"] = worker
-            worker.work()
-            self._logs.extend(worker.logs())
+        worker = None
+        try:
+            with self._worker_cls(
+                self._server, queue, PROCEDURE.WORKER.QUEUE_TIMEOUT_S, env
+            ) as worker:
+                with self.lock:
+                    self._active_workers[queue]["worker"] = worker
+                worker.work()
+        finally:
+            if worker is not None:
+                logs = worker.logs()
+                self._logs.extend(logs)
+                for line in logs:
+                    logger.info(f"Procedure worker '{queue}': {line}")
 
     def _wait_for_all_futures(self):
         with self.lock:
@@ -195,7 +206,7 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         self._conn.shutdown()
         # cancel futures by hand to give us the opportunity to detach them from redis if they have started
         with self.lock:
-            for entry in self._active_workers.values():
+            for entry in list(self._active_workers.values()):
                 cancelled = entry["future"].cancel()
                 if not cancelled:
                     # unblock any waiting workers and let them shutdown
@@ -292,6 +303,20 @@ class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureEx
         def cleanup_worker(fut):
             with self.lock:
                 logger.debug(f"cleaning up worker {fut} for queue {queue}...")
+                error = None if fut.cancelled() else fut.exception()
+                if error is not None:
+                    pending = self._helper.get.exec_queue(queue)
+                    running = [
+                        msg for msg in self._helper.get.running_procedures() if msg.queue == queue
+                    ]
+                    self._helper.move.execution_queue_to_unhandled(queue)
+                    for msg in running:
+                        self._helper.push.unhandled(queue, msg)
+                    for msg in pending + running:
+                        self._helper.status_update(
+                            msg.execution_id, "Finished", error=f"Procedure worker failed: {error}"
+                        )
+                    self._helper.notify_watchers(queue, "unhandled")
                 self._helper.remove_from_active.by_queue(queue)
                 self._run_callbacks(queue)
                 self._helper.notify_watchers(queue, "execution")

{bec_server-4.3.0 → bec_server-4.3.1}/bec_server/procedures/oop_worker_base.py

@@ -59,7 +59,7 @@ class OutOfProcessWorkerBase(ProcedureWorker):
         minimum necessary, or things which are only necessary for the functioning of the worker,
         and other information should be passed through redis"""
         return {
-            "redis_server": f"{PROCEDURE.REDIS_HOST}:{self._conn.port}",
+            "redis_server": self._redis_server,
             "queue": self._queue,
             "timeout_s": str(self._lifetime_s),
         }
@@ -70,7 +70,7 @@ class OutOfProcessWorkerBase(ProcedureWorker):
         # on timeout check if container is still running
 
         status_update = None
-        while not self._ending_or_ended():
+        while not self._aborted.is_set():
             status_update = self._conn.blocking_list_pop(
                 MessageEndpoints.procedure_worker_status_update(self._queue), timeout_s=0.2
             )
@@ -82,10 +82,17 @@ class OutOfProcessWorkerBase(ProcedureWorker):
                 logger.info(
                     f"Procedure worker '{self._queue}' status update: {status_update.status.name}"
                 )
+            elif self._ending_or_ended():
+                break
             # TODO: we probably do want to handle some kind of timeout here but we don't know how
             # long a running procedure should actually take - it could theoretically be infinite
         if self.status != ProcedureWorkerStatus.FINISHED:
             self.status = ProcedureWorkerStatus.DEAD
+            if not self._aborted.is_set():
+                logs = "\n".join(self.logs())
+                raise ProcedureWorkerError(
+                    f"Procedure worker for queue {self._queue} exited unexpectedly.\n{logs}"
+                )
 
 
 def _default_env() -> dict[str, str]:
@@ -230,10 +237,21 @@ def _main(
             helper.remove_from_active.by_exec_id(item.execution_id)
 
 
-def main():
-    """Replaces the main contents of Worker.work() - should be called as the container entrypoint or command"""
+def main(*, output_to_redis: bool = True) -> None:
+    """Run the worker entry point.
+
+    Args:
+        output_to_redis (bool): Redirect procedure output to Redis in this process. Subprocess
+            workers disable this because their parent streams both output pipes instead.
+    """
 
     env, helper, client, conn = setup(get_env())
+    if not output_to_redis:
+        try:
+            _main(env, helper, client, conn)
+        finally:
+            conn.shutdown()
+        return
     logger_connector = RedisConnector(env["redis_server"])
     output_diverter = RedisOutputDiverter(logger_connector, env["queue"])
     with redirect_stdout(output_diverter):

bec_server-4.3.1/bec_server/procedures/subprocess_worker.py

@@ -0,0 +1,144 @@
+import os
+import subprocess
+import sys
+from codecs import getincrementaldecoder
+from collections import deque
+from signal import SIGINT
+from threading import Lock, Thread
+
+from bec_lib.endpoints import MessageEndpoints
+from bec_lib.logger import bec_logger
+from bec_lib.messages import RawMessage
+from bec_lib.redis_connector import RedisConnector
+from bec_server.procedures.constants import ProcedureWorkerError
+from bec_server.procedures.oop_worker_base import PROCESS_TIMEOUT, OutOfProcessWorkerBase, main
+
+logger = bec_logger.logger
+
+
+class SubProcessWorker(OutOfProcessWorkerBase):
+    """Run procedures in a subprocess, streaming stdout and stderr to Redis."""
+
+    WORKER_FILE = __file__
+
+    def _setup_execution_environment(self):
+        # Match the UTF-8 decoder in the parent even when the host uses another encoding.
+        env: dict[str, str] = {"PYTHONIOENCODING": "utf-8"} | self._worker_environment()
+        first_start = not hasattr(self, "_output_logs")
+        self._output_error_logged = False
+        if first_start:
+            self._output_logs: deque[str] = deque(maxlen=10000)
+            self._output_lock = Lock()
+        output_conn = RedisConnector(
+            [self._redis_server], name=f"ProcedureWorker-{self._queue} output"
+        )
+        try:
+            if first_start:
+                output_conn.delete(MessageEndpoints.procedure_logs(self._queue))
+            self._process = subprocess.Popen(
+                [sys.executable, "-u", self.WORKER_FILE],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                bufsize=0,
+                env=os.environ | env | self._additional_env,
+            )
+        except BaseException:
+            output_conn.shutdown()
+            raise
+        self._output_thread = Thread(
+            target=self._capture_output,
+            args=(self._process, output_conn),
+            name=f"ProcedureWorker-{self._queue} output",
+            daemon=True,
+        )
+        self._output_thread.start()
+        logger.success("Subprocess Procedure Worker started.")
+        self._ending: bool = False
+
+    def _capture_output(self, process: subprocess.Popen, conn: RedisConnector) -> None:
+        """Drain both streams even if Redis rejects an output write."""
+        decoder = getincrementaldecoder("utf-8")(errors="replace")
+        try:
+            with process.stdout as output:
+                while True:
+                    chunk = output.read(4096)
+                    if not chunk:
+                        self._publish_output(decoder.decode(b"", final=True), conn)
+                        break
+                    self._publish_output(decoder.decode(chunk), conn)
+        finally:
+            conn.shutdown()
+
+    def _publish_output(self, text: str, conn: RedisConnector) -> None:
+        if not text:
+            return
+        with self._output_lock:
+            self._output_logs.append(text)
+        try:
+            conn.xadd(
+                MessageEndpoints.procedure_logs(self._queue),
+                {"data": RawMessage(data=text)},
+                max_size=10000,
+                expire=3600,
+            )
+        except Exception as exc:
+            if not self._output_error_logged:
+                logger.error(f"Failed to publish procedure output: {exc}")
+            self._output_error_logged = True
+
+    def _join_output(self) -> None:
+        if hasattr(self, "_output_thread"):
+            self._output_thread.join(timeout=PROCESS_TIMEOUT)
+            if self._output_thread.is_alive():
+                raise ProcedureWorkerError("Subprocess output reader failed to stop")
+
+    def _cleanup_execution_environment(self) -> None:
+        try:
+            super()._cleanup_execution_environment()
+        finally:
+            self._join_output()
+
+    def _ending_or_ended(self):
+        return self._ending or (self._process.poll() is not None)
+
+    def _kill_process(self):
+        """Attempts to end the process gently by first sending SIGINT, which is handled by the BECIpythonClient to
+        gracefully shut down, and end any scans it has started. See:
+        bec_ipython_client/main.py#L125 and bec_ipython_client/signals.py#L103
+        If the process fails to shut down naturally, kills it unceremoniously. This will leave the running scan running
+        on the ScanServer.
+        """
+        if not self._ending_or_ended():
+            self._ending = True
+            self._process.send_signal(SIGINT)
+            self._process.send_signal(SIGINT)
+            try:
+                self._process.wait(PROCESS_TIMEOUT)
+            except subprocess.TimeoutExpired:
+                logger.warning(
+                    f"Subprocess Procedure Worker failed to exit in {PROCESS_TIMEOUT} s. Killing."
+                )
+                self._process.kill()
+                self._process.wait(PROCESS_TIMEOUT)
+
+    def abort_execution(self, execution_id: str):
+        """Abort the execution with the given id. Has no effect if the given ID is not the current job"""
+        if execution_id == self._current_execution_id:
+            self._kill_process()
+            self._helper.remove_from_active.by_exec_id(execution_id)
+            logger.info(
+                f"Aborting execution {execution_id}, restarting worker for queue: {self._queue}"
+            )
+            self._join_output()
+            self._setup_execution_environment()
+
+    def logs(self) -> list[str]:
+        """Return captured output without consuming it or waiting for a live process."""
+        if self._process.poll() is not None:
+            self._join_output()
+        with self._output_lock:
+            return "".join(self._output_logs).splitlines()
+
+
+if __name__ == "__main__":
+    main(output_to_redis=False)

{bec_server-4.3.0 → bec_server-4.3.1}/bec_server/procedures/worker_base.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from threading import Event
+from threading import Event, Lock
 
 from bec_lib.endpoints import MessageEndpoints
 from bec_lib.logger import bec_logger
@@ -52,15 +52,27 @@ class ProcedureWorker(ABC):
         self.client_id = self._conn.client_id()
         self._current_execution_id: str | None = None
         self._aborted = Event()
+        self._shutdown_lock = Lock()
 
-        self._setup_execution_environment()
+        try:
+            self._setup_execution_environment()
+        except BaseException:
+            self._conn.shutdown()
+            raise
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self._conn.shutdown()
-        self._kill_process()
+        self._cleanup_execution_environment()
+
+    def _cleanup_execution_environment(self) -> None:
+        """Serialize abort and context teardown, always stopping the external process."""
+        with self._shutdown_lock:
+            try:
+                self._conn.shutdown()
+            finally:
+                self._kill_process()
 
     @abstractmethod
     def _kill_process(self):
@@ -86,8 +98,7 @@ class ProcedureWorker(ABC):
     def abort(self):
         """Abort the entire worker"""
         self._aborted.set()
-        self._conn.shutdown()
-        self._kill_process()
+        self._cleanup_execution_environment()
 
     @abstractmethod
     def abort_execution(self, execution_id: str):

{bec_server-4.3.0 → bec_server-4.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "bec-server"
-version = "4.3.0"
+version = "4.3.1"
 description = "BEC server"
 requires-python = ">=3.11"
 classifiers = [

{bec_server-4.3.0 → bec_server-4.3.1}/tests/tests_actors/test_actors.py

@@ -20,8 +20,8 @@ from bec_server.test.helpers import wait_until
 
 @pytest.fixture
 def fakeredis_config():
-    redis_config = "localhost", 44556
-    server = TcpFakeServer(redis_config)
+    server = TcpFakeServer(("127.0.0.1", 0))
+    redis_config = server.server_address
     t = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
     t.start()
     yield redis_config
@@ -122,6 +122,7 @@ def test_subscription_actor_unsubs():
 def test_polling_actor_inline(fakeredis_config):
     host, port = fakeredis_config
     redis = {"host": host, "port": port}
+    BECClient._reset_singleton()
     client: BECClient = _create_client(BecClientType.BECClient, redis=redis)  # type: ignore
     client.start()
     redis = f"{host}:{port}"
@@ -140,6 +141,7 @@ def test_polling_actor_inline(fakeredis_config):
         actor.stop_event.set()
         actor_thread.join()
         client.shutdown()
+        BECClient._reset_singleton()
 
 
 def test_actor_procedure_happy_path():
@@ -263,3 +265,26 @@ def test_bl_state_actor_cancelled_startup(ready, stop_before_run, threads_check)
             mock_client.beamline_states.ready = True
             actor.stop_event.set()
             t.join(timeout=2)
+
+
+@patch("bec_server.actors.worker.actor_procedure")
+@patch("bec_server.actors.worker.push_status")
+@patch("bec_server.actors.worker.setup")
+@patch("bec_server.actors.worker.get_actor_env", return_value={})
+@patch("bec_server.actors.worker.get_env")
+def test_actor_worker_reports_finished(
+    get_env_mock, actor_env_mock, setup_mock, push_mock, procedure_mock
+):
+    from bec_server.actors.worker import main
+
+    conn = MagicMock()
+    setup_mock.return_value = (
+        {"redis_server": "localhost:6379", "queue": "test"},
+        None,
+        MagicMock(),
+        conn,
+    )
+    with patch("bec_server.actors.worker.logger"):
+        main()
+    assert push_mock.call_args.args == (conn, "test", ProcedureWorkerStatus.FINISHED)
+    conn.shutdown.assert_called_once()

{bec_server-4.3.0 → bec_server-4.3.1}/tests/tests_scan_server/test_container_utils.py

@@ -1,3 +1,4 @@
+from subprocess import CompletedProcess
 from unittest.mock import ANY, MagicMock, patch
 
 import pytest
@@ -217,3 +218,37 @@ def test_cli_get_state(cli_utils_with_fake_container_json: PodmanCliUtils):
     assert cli_utils_with_fake_container_json.state(
         "13826d25a737b733a5d87975b50a9c1efb5d087365d956cb7db2ccbe6aca07c4"
     ) == PodmanContainerStates("running")
+
+
+def test_cli_logs_include_stdout_and_stderr(cli_utils):
+    utils, run = cli_utils
+    run.return_value.stdout = b"startup output\n"
+    run.return_value.stderr = b"startup error\n"
+    assert utils.logs("test") == ["startup output", "startup error"]
+
+
+def test_cli_run_ignores_missing_previous_container(cli_utils):
+    utils, run_mock = cli_utils
+    run_mock.reset_mock()
+
+    def run_command(command, **kwargs):
+        missing_container = command[1] == "rm" and "--ignore" not in command
+        return CompletedProcess(
+            command,
+            returncode=1 if missing_container else 0,
+            stdout=b"worker-id\n",
+            stderr=b"No such container" if missing_container else b"",
+        )
+
+    run_mock.side_effect = run_command
+    container_id = utils.run(
+        "worker-image",
+        {"redis_server": "localhost:6379", "queue": "test", "timeout_s": "10"},
+        [],
+        "bec-procedure-worker",
+        container_name="new-worker",
+    )
+
+    assert container_id == "worker-id"
+    run_mock.assert_any_call(["podman", "rm", "-f", "--ignore", "new-worker"], capture_output=True)
+    assert run_mock.call_args.args[0][:2] == ["podman", "run"]

{bec_server-4.3.0 → bec_server-4.3.1}/tests/tests_scan_server/test_procedure_container_worker.py

@@ -1,7 +1,7 @@
 import os
 import time
 from itertools import repeat
-from threading import Thread
+from threading import Event, Thread
 from time import sleep
 from unittest.mock import MagicMock, call, patch
 
@@ -25,7 +25,7 @@ def test_container_worker_init(redis_mock):
     redis_mock().port = "port"
     worker = ContainerProcedureWorker(server="server:port", queue="test_queue", lifetime_s=1)
     assert worker._worker_environment() == {
-        "redis_server": "redis:port",
+        "redis_server": "server:port",
         "queue": "test_queue",
         "timeout_s": "1",
     }
@@ -185,3 +185,120 @@ def test_main_running_newly_registered_proc(redis_mock, logger_mock):
     logger_mock.debug.assert_has_calls([call(f"running task {PROC_NAME}")], any_order=True)
     function_recorder.assert_called_once_with("a", b="c")
     logger_mock.success.assert_called_with("Procedure runner shutting down")
+
+
+@patch("bec_server.procedures.container_worker.get_backend")
+@patch("bec_server.procedures.worker_base.RedisConnector")
+def test_container_worker_uses_supplied_environment(redis_mock, backend_mock):
+    worker = ContainerProcedureWorker(
+        server="127.0.0.1:20772",
+        queue="test",
+        env={"redis_server": "host.containers.internal:20772"},
+    )
+    assert backend_mock().run.call_args.args[1]["redis_server"] == "host.containers.internal:20772"
+    assert worker._worker_environment()["redis_server"] == "127.0.0.1:20772"
+
+
+@patch("bec_server.procedures.container_worker.get_backend")
+@patch("bec_server.procedures.worker_base.RedisConnector")
+def test_container_worker_setup_failure_closes_connector(redis_mock, backend_mock):
+    backend_mock().run.side_effect = RuntimeError("missing pod")
+    with pytest.raises(RuntimeError, match="missing pod"):
+        ContainerProcedureWorker(server="localhost:6379", queue="test")
+    redis_mock().shutdown.assert_called_once()
+
+
+@patch("bec_server.procedures.container_worker.get_backend")
+@patch("bec_server.procedures.worker_base.RedisConnector")
+def test_container_worker_early_exit_reports_logs(redis_mock, backend_mock):
+    from bec_server.procedures.constants import ProcedureWorkerError
+
+    redis_mock().blocking_list_pop.return_value = None
+    backend_mock().state.return_value = PodmanContainerStates.EXITED
+    backend_mock().logs.return_value = ["Cannot connect to Redis"]
+    worker = ContainerProcedureWorker(server="localhost:6379", queue="test")
+    with pytest.raises(ProcedureWorkerError, match="Cannot connect to Redis"):
+        worker.work()
+    assert worker.status == ProcedureWorkerStatus.DEAD
+
+
+@patch("bec_server.procedures.container_worker.get_backend")
+@patch("bec_server.procedures.worker_base.RedisConnector")
+def test_container_worker_drains_final_status_before_exit(redis_mock, backend_mock):
+    redis_mock().blocking_list_pop.side_effect = [
+        ProcedureWorkerStatusMessage(worker_queue="test", status=ProcedureWorkerStatus.FINISHED),
+        None,
+    ]
+    backend_mock().state.return_value = PodmanContainerStates.EXITED
+    worker = ContainerProcedureWorker(server="localhost:6379", queue="test")
+    worker.work()
+    assert worker.status == ProcedureWorkerStatus.FINISHED
+
+
+@patch("bec_server.procedures.container_worker.get_backend")
+@patch("bec_server.procedures.worker_base.RedisConnector")
+def test_abort_and_context_exit_serialize_connector_shutdown(redis_mock, backend_mock):
+    worker = ContainerProcedureWorker(server="localhost:6379", queue="test")
+    closing = Event()
+    release = Event()
+    exit_started = Event()
+    concurrent_close = Event()
+    errors = []
+
+    def close_connector():
+        if closing.is_set():
+            concurrent_close.set()
+            raise RuntimeError("concurrent Redis connector shutdown")
+        closing.set()
+        try:
+            assert release.wait(2)
+        finally:
+            closing.clear()
+
+    def abort():
+        try:
+            worker.abort()
+        except BaseException as exc:
+            errors.append(exc)
+
+    def exit_context():
+        exit_started.set()
+        try:
+            worker.__exit__(None, None, None)
+        except BaseException as exc:
+            errors.append(exc)
+
+    redis_mock().shutdown.side_effect = close_connector
+    worker._kill_process = MagicMock()
+    abort_thread = Thread(target=abort)
+    exit_thread = Thread(target=exit_context)
+    abort_thread.start()
+    try:
+        assert closing.wait(2)
+        exit_thread.start()
+        assert exit_started.wait(2)
+        assert not concurrent_close.wait(0.1)
+    finally:
+        release.set()
+        abort_thread.join(2)
+        if exit_thread.ident is not None:
+            exit_thread.join(2)
+    assert not abort_thread.is_alive()
+    assert not exit_thread.is_alive()
+    assert errors == []
+    worker._kill_process.assert_called()
+
+
+@patch("bec_server.procedures.container_worker.get_backend")
+@patch("bec_server.procedures.worker_base.RedisConnector")
+@pytest.mark.parametrize("abort", [True, False])
+def test_connector_shutdown_failure_still_stops_worker(redis_mock, backend_mock, abort):
+    worker = ContainerProcedureWorker(server="localhost:6379", queue="test")
+    redis_mock().shutdown.side_effect = RuntimeError("failed to close Redis")
+    worker._kill_process = MagicMock()
+    with pytest.raises(RuntimeError, match="failed to close Redis"):
+        if abort:
+            worker.abort()
+        else:
+            worker.__exit__(None, None, None)
+    worker._kill_process.assert_called_once()