bec-server 4.3.1__tar.gz → 4.3.2__tar.gz

{bec_server-4.3.1 → bec_server-4.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: bec-server
-Version: 4.3.1
+Version: 4.3.2
 Summary: BEC server
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

{bec_server-4.3.1 → bec_server-4.3.2}/bec_server/procedures/constants.py

@@ -18,6 +18,7 @@ class OopWorkerEnv(TypedDict):
     queue: str
     timeout_s: str
     client_class: NotRequired[BecClientType]
+    worker_id: NotRequired[str]
 
 
 P = ParamSpec("P")
@@ -33,7 +34,25 @@ class BecProcedure(Protocol[P]):
     def __call__(self, *args: P.args, **kwargs: P.kwargs) -> None: ...
 
 
-class ProcedureWorkerError(RuntimeError): ...
+class ProcedureWorkerFailure(str, Enum):
+    """Fixed, credential-free descriptions suitable for public execution status."""
+
+    STARTUP = "Procedure worker startup failed"
+    AUTHENTICATION = "Procedure worker authentication failed"
+    EXITED = "Procedure worker exited unexpectedly"
+    INPUT_CLOSED = "Procedure worker input closed unexpectedly"
+    ABORTED = "Procedure worker was aborted"
+
+
+class ProcedureWorkerError(RuntimeError):
+    """Worker failure with an optional safe category for public status."""
+
+    def __init__(self, *args: object, failure: ProcedureWorkerFailure | None = None) -> None:
+        """Default to the public failure description unless internal details are supplied."""
+        if not args and failure is not None:
+            args = (failure.value,)
+        super().__init__(*args)
+        self.failure = failure
 
 
 class WorkerAlreadyExists(ProcedureWorkerError): ...
@@ -48,7 +67,7 @@ class NoImage(ProcedureWorkerError): ...
 @dataclass(frozen=True)
 class _WORKER:
     MAX_WORKERS = 10
-    QUEUE_TIMEOUT_S = 10
+    QUEUE_TIMEOUT_S = 60
     DEFAULT_QUEUE = "primary"
 
 

{bec_server-4.3.1 → bec_server-4.3.2}/bec_server/procedures/container_utils.py

@@ -10,7 +10,7 @@ from typing import Iterator, Literal, cast
 
 from podman import PodmanClient
 from podman.domain.containers import Container
-from podman.errors import APIError
+from podman.errors import APIError, NotFound
 
 from bec_lib.logger import bec_logger
 from bec_server.procedures.constants import (
@@ -81,6 +81,67 @@ class _PodmanUtilsBase(ContainerCommandBackend):
             tag=f"{PROCEDURE.CONTAINER.IMAGE_NAME}:v{PROCEDURE.BEC_VERSION}",
         )
 
+    def run_interactive(
+        self,
+        image_tag: str,
+        environment: OopWorkerEnv,
+        volumes: list[VolumeSpec],
+        command: str,
+        pod_name: str | None = None,
+        container_name: str | None = None,
+    ) -> subprocess.Popen:
+        """Attach process pipes so private worker bootstrap never enters Podman metadata.
+
+        Args:
+            image_tag (str): Worker image.
+            environment (OopWorkerEnv): Non-secret worker settings.
+            volumes (list[VolumeSpec]): Code mounts.
+            command (str): Worker entry point.
+            pod_name (str | None): Existing Podman pod.
+            container_name (str | None): Unique worker container name.
+        Returns:
+            subprocess.Popen: Attached process with stdin and separate output pipes.
+        """
+        prefix = self._cli_prefix()
+        args = [*prefix, "create", "-i"]
+        args.extend(_multi_args_from_dict("-e", environment))
+        for volume in volumes:
+            args.extend(
+                [
+                    "-v",
+                    f"{volume['source']}:{volume['target']}{':ro' if volume['read_only'] else ''}",
+                ]
+            )
+        if pod_name:
+            args.extend(["--pod", pod_name])
+        if container_name:
+            args.extend(["--name", container_name])
+        # Create synchronously: cancelling the attached start cannot leave a later
+        # `podman run` creating an orphan container after cleanup has returned.
+        container_id = _run_and_capture_error(*args, image_tag, command).stdout.decode().strip()
+        try:
+            return subprocess.Popen(
+                [*prefix, "start", "--attach", "--interactive", container_id],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                bufsize=0,
+            )
+        except BaseException:
+            self.remove(container_id)
+            raise
+
+    def _cli_prefix(self) -> list[str]:
+        return ["podman", "--url", self.uri] if isinstance(self, PodmanApiUtils) else ["podman"]
+
+    def remove(self, id: str) -> None:
+        """Remove a stopped or starting worker container, preventing a later start.
+
+        Args:
+            id (str): Container name or ID.
+        """
+        _run_and_capture_error(*self._cli_prefix(), "rm", "--force", "--ignore", id)
+
 
 class PodmanApiOutput(ContainerCommandOutput):
     def __init__(self, command_output: Iterator[bytes]):
@@ -157,7 +218,10 @@ class PodmanApiUtils(_PodmanUtilsBase):
 
     def state(self, id: str) -> PodmanContainerStates | None:
         with PodmanClient(base_url=self.uri) as client:
-            status = client.containers.get(id).status
+            try:
+                status = client.containers.get(id).status
+            except NotFound:
+                return None
             if status == "unknown":
                 return None
             return PodmanContainerStates(status)
@@ -259,7 +323,11 @@ class PodmanCliUtils(_PodmanUtilsBase):
 
     def state(self, id: str) -> PodmanContainerStates | None:
         for container in self._podman_ls_json():
-            if container["Id"] == id or container["Id"].startswith(id):
+            if (
+                container["Id"] == id
+                or container["Id"].startswith(id)
+                or id in container.get("Names", [])
+            ):
                 if names := container.get("Names"):
                     logger.debug(f"Container {names[0]} status: {container['State']}")
                 return PodmanContainerStates(container["State"])

bec_server-4.3.2/bec_server/procedures/container_worker.py

@@ -0,0 +1,152 @@
+import time
+
+from bec_lib.logger import bec_logger
+from bec_lib.redis_connector import RedisConnector
+from bec_server.procedures.constants import PROCEDURE, PodmanContainerStates
+from bec_server.procedures.container_utils import get_backend
+from bec_server.procedures.oop_worker_base import (
+    main,  # temporarily needed before update of base image on release
+)
+from bec_server.procedures.oop_worker_base import PROCESS_TIMEOUT, OutOfProcessWorkerBase
+from bec_server.procedures.protocol import ContainerCommandBackend
+
+logger = bec_logger.logger
+
+
+class ContainerProcedureWorker(OutOfProcessWorkerBase):
+    """A worker which runs scripts in a container with a full BEC environment,
+    mounted from the filesystem, and only access to Redis"""
+
+    # The Podman client is a thin wrapper around the libpod API
+    # documented at https://docs.podman.io/en/latest/_static/api.html
+    # which is more detailed than the podman-py documentation
+
+    def _setup_execution_environment(self):
+        self._backend: ContainerCommandBackend = get_backend()
+        image_tag = f"{PROCEDURE.CONTAINER.IMAGE_NAME}:v{PROCEDURE.BEC_VERSION}"
+        self.container_name = (
+            f"bec_procedure_{PROCEDURE.BEC_VERSION}_{self._worker_id or self._queue}"
+        )
+        if not self._backend.image_exists(image_tag):
+            self._backend.build_worker_image()
+        environment = self._worker_environment() | self._additional_env
+        if self._worker_id is not None:
+            self._container_id = self.container_name
+            self._process = self._backend.run_interactive(
+                image_tag,
+                environment,
+                [
+                    {
+                        "source": str(PROCEDURE.CONTAINER.DEPLOYMENT_PATH),
+                        "target": "/bec",
+                        "type": "bind",
+                        "read_only": True,
+                    }
+                ],
+                PROCEDURE.CONTAINER.COMMAND,
+                pod_name=PROCEDURE.CONTAINER.POD_NAME,
+                container_name=self.container_name,
+            )
+            output_conn = RedisConnector([self._redis_server], name="Procedure container output")
+            try:
+                if self._control_credentials is not None:
+                    output_conn.authenticate(
+                        username=self._control_credentials["username"] or "default",
+                        password=self._control_credentials["token"],
+                    )
+                self._start_output_capture(output_conn)
+            except BaseException:
+                output_conn.shutdown()
+                self._kill_process()
+                raise
+            return
+        self._container_id = self._backend.run(
+            image_tag,
+            environment,
+            [
+                {
+                    "source": str(PROCEDURE.CONTAINER.DEPLOYMENT_PATH),
+                    "target": "/bec",
+                    "type": "bind",
+                    "read_only": True,
+                }
+            ],
+            PROCEDURE.CONTAINER.COMMAND,
+            pod_name=PROCEDURE.CONTAINER.POD_NAME,
+            container_name=self.container_name,
+        )
+
+    def _ending_or_ended(self):
+        if self._worker_id is not None:
+            return self._process.poll() is not None
+        return self._backend.state(self._container_id) in [
+            PodmanContainerStates.EXITED,
+            PodmanContainerStates.STOPPED,
+            PodmanContainerStates.STOPPING,
+        ]
+
+    def _kill_process(self):
+        if self._worker_id is not None:
+            # The attached CLI can fail while the container is still running.
+            terminal = (
+                None,
+                PodmanContainerStates.CREATED,
+                PodmanContainerStates.EXITED,
+                PodmanContainerStates.STOPPED,
+            )
+            if self._backend.state(self.container_name) not in terminal:
+                self._backend.interrupt(self.container_name)
+                deadline = time.monotonic() + PROCESS_TIMEOUT
+                while time.monotonic() < deadline:
+                    if self._backend.state(self.container_name) in terminal:
+                        break
+                    time.sleep(0.1)
+                else:
+                    self._backend.kill(self.container_name)
+                deadline = time.monotonic() + PROCESS_TIMEOUT
+                while self._backend.state(self.container_name) not in terminal:
+                    if time.monotonic() >= deadline:
+                        raise RuntimeError("Procedure container failed to stop")
+                    time.sleep(0.1)
+            # Creation completed before the launcher was started, so removal also
+            # prevents an attached start that has not reached the engine yet.
+            self._backend.remove(self.container_name)
+            if self._process.stdin is not None:
+                self._process.stdin.close()
+            try:
+                self._process.wait(PROCESS_TIMEOUT)
+            except Exception:
+                self._process.kill()
+                self._process.wait(PROCESS_TIMEOUT)
+            return
+        if not self._ending_or_ended():
+            self._backend.interrupt(self.container_name)
+            start = time.monotonic()
+            while time.monotonic() < start + PROCESS_TIMEOUT:
+                if self._ending_or_ended():
+                    return
+                time.sleep(0.2)
+            logger.warning(
+                f"Procedure worker {self._container_id} for queue {self._queue} failed to shut down, killing."
+            )
+            self._backend.kill(self.container_name)
+
+    def abort_execution(self, execution_id: str):
+        """Abort the execution with the given id. Has no effect if the given ID is not the current job"""
+        if execution_id == self._current_execution_id:
+            self._backend.kill(self._container_id)
+            self._helper.remove_from_active.by_exec_id(execution_id)
+            logger.info(
+                f"Aborting execution {execution_id}, restarting worker for queue: {self._queue}"
+            )
+            self._setup_execution_environment()
+
+    def logs(self):
+        if self._worker_id is not None:
+            if self._process.poll() is not None:
+                self._join_output()
+            with self._output_lock:
+                return "".join(self._output_logs).splitlines()
+        if self._container_id is None:
+            return [""]
+        return self._backend.logs(self._container_id)

{bec_server-4.3.1 → bec_server-4.3.2}/bec_server/procedures/manager.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import atexit
+import re
 from abc import ABC, abstractmethod
 from collections import deque
 from concurrent import futures
 from concurrent.futures import Future, ThreadPoolExecutor
 from threading import RLock
-from typing import TYPE_CHECKING, Any, Callable, Generic, Protocol, TypedDict, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Generic, NotRequired, Protocol, TypedDict, TypeVar
 
 from pydantic import ValidationError
 
@@ -15,7 +16,6 @@ from bec_lib.logger import bec_logger
 from bec_lib.messages import (
     AvailableResourceMessage,
     BECMessage,
-    ProcedureAbortMessage,
     ProcedureClearUnhandledMessage,
     ProcedureExecutionMessage,
     ProcedureQNotifMessage,
@@ -23,10 +23,10 @@ from bec_lib.messages import (
     ProcedureWorkerStatus,
     RequestResponseMessage,
 )
-from bec_lib.procedures.helper import BackendProcedureHelper
 from bec_lib.redis_connector import MessageObject, RedisConnector
 from bec_server.procedures import procedure_registry
 from bec_server.procedures.constants import PROCEDURE, WorkerAlreadyExists
+from bec_server.procedures.queue_manager import QueueWorkerPools
 
 if TYPE_CHECKING:
     from bec_server.procedures.worker_base import ProcedureWorker
@@ -37,6 +37,7 @@ logger = bec_logger.logger
 class ProcedureWorkerEntry(TypedDict):
     worker: ProcedureWorker | None
     future: Future
+    abort_requested: NotRequired[bool]
 
 
 class RedisCredentials(TypedDict):
@@ -75,6 +76,8 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         worker_type: type[ProcedureWorker],
         thread_prefix: str = "user_procedure_",
         redis_credentials: RedisCredentials | None = None,
+        *,
+        _defer_startup: bool = False,
     ):
         """Watches the request queue and pushes to worker execution queues. Manages
         instantiating and cleaning up after workers.
@@ -114,8 +117,9 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         self._server = f"{self._conn.host}:{self._conn.port}"
 
         self._reply_ep: EndpointInfo
-        self._register_endpoints()
-        self._publish_available()
+        if not _defer_startup:
+            self._register_endpoints()
+            self._publish_available()
         logger.success("Done initialising procedure manager.")
 
     def _ack(self, accepted: bool, msg: str, exec_id: str):
@@ -135,7 +139,7 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         Args:
             msg (dict[str, Any]): dict corresponding to a ProcedureRequestMessage"""
 
-        logger.debug(f"Procedure manager got request message {msg}")
+        logger.debug(f"Procedure manager got queue request on topic {msg.topic}")
         if (message := self._validate_request(msg.value)) is None:
             return
         exec_msg = self._respond_to_valid_request(message)
@@ -164,8 +168,13 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
                 self._server, queue, PROCEDURE.WORKER.QUEUE_TIMEOUT_S, env
             ) as worker:
                 with self.lock:
-                    self._active_workers[queue]["worker"] = worker
-                worker.work()
+                    entry = self._active_workers[queue]
+                    entry["worker"] = worker
+                    abort_requested = entry.get("abort_requested", False)
+                if abort_requested:
+                    worker.abort()
+                else:
+                    worker.work()
         finally:
             if worker is not None:
                 logs = worker.logs()
@@ -173,9 +182,22 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
                 for line in logs:
                     logger.info(f"Procedure worker '{queue}': {line}")
 
-    def _wait_for_all_futures(self):
+    def _abort_workers(
+        self, entries: list[ProcedureWorkerEntry], before_wait: Callable[[], None] | None = None
+    ) -> None:
+        """Stop selected workers, optionally draining queues before waiting for startup."""
         with self.lock:
-            futs = list(entry["future"] for entry in self._active_workers.values())
+            for entry in entries:
+                entry["abort_requested"] = True
+            workers = [entry["worker"] for entry in entries if entry["worker"] is not None]
+        for entry in entries:
+            entry["future"].cancel()
+        for worker in workers:
+            worker.abort()
+        if before_wait is not None:
+            before_wait()
+        # A cancelled future need not have been picked up by an executor thread yet.
+        futs = [entry["future"] for entry in entries if not entry["future"].cancelled()]
         futures.wait(futs, timeout=PROCEDURE.MANAGER_SHUTDOWN_TIMEOUT_S)
 
     def active_workers(self) -> list[str]:
@@ -204,16 +226,9 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         logger.debug(f"shutting down {self.__class__.__name__}")
         self._unregister_endpoints()
         self._conn.shutdown()
-        # cancel futures by hand to give us the opportunity to detach them from redis if they have started
         with self.lock:
-            for entry in list(self._active_workers.values()):
-                cancelled = entry["future"].cancel()
-                if not cancelled:
-                    # unblock any waiting workers and let them shutdown
-                    if worker := entry["worker"]:
-                        # redis unblock executor.client_id
-                        worker.abort()
-        self._wait_for_all_futures()
+            entries = list(self._active_workers.values())
+        self._abort_workers(entries)
         self.executor.shutdown()
 
     @abstractmethod
@@ -252,24 +267,15 @@ class ProcedureManagerBase(ABC, Generic[_ReqMsgT, _ExecMsgT]):
         """Returns a function which will be attached as a callback to finishing workers."""
 
 
-class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureExecutionMessage]):
-    def __init__(
-        self,
-        redis: str,
-        worker_type: type[ProcedureWorker],
-        thread_prefix: str = "user_procedure_",
-        redis_credentials: RedisCredentials | None = None,
-    ):
-        super().__init__(redis, worker_type, thread_prefix, redis_credentials)
-        self._messages_by_ids: dict[str, ProcedureExecutionMessage] = {}
-        self._helper = BackendProcedureHelper(self._conn, monitor_responses=False)
-        self._startup()
+class ProcedureManager(
+    QueueWorkerPools, ProcedureManagerBase[ProcedureRequestMessage, ProcedureExecutionMessage]
+):
 
     def _define_endpoints(self):
         self._reply_ep = MessageEndpoints.procedure_request_response()
         self._abort_ep = MessageEndpoints.procedure_abort()
         self._clear_uh_ep = MessageEndpoints.procedure_clear_unhandled()
-        self._request_ep = MessageEndpoints.procedure_request()
+        self._request_ep = MessageEndpoints.procedure_request("*")
         self._q_notif_ep = MessageEndpoints.procedure_queue_notif()
         self._exec_ep = MessageEndpoints.procedure_execution
 
@@ -287,7 +293,7 @@ class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureEx
     def _register_endpoints(self):
         self._conn.register(self._abort_ep, None, self._process_abort)
         self._conn.register(self._clear_uh_ep, None, self._process_clear_unhandled)
-        self._conn.register(self._request_ep, None, self._process_queue_request)
+        self._conn.register(patterns=self._request_ep, cb=self._procedure_request_callback)
         self._conn.register(self._q_notif_ep, None, self._publish_metrics)
         self._conn.set(
             MessageEndpoints.available_procedures(),
@@ -299,33 +305,32 @@ class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureEx
             ),
         )
 
-    def _cleanup_worker_function(self, queue: str):
-        def cleanup_worker(fut):
-            with self.lock:
-                logger.debug(f"cleaning up worker {fut} for queue {queue}...")
-                error = None if fut.cancelled() else fut.exception()
-                if error is not None:
-                    pending = self._helper.get.exec_queue(queue)
-                    running = [
-                        msg for msg in self._helper.get.running_procedures() if msg.queue == queue
-                    ]
-                    self._helper.move.execution_queue_to_unhandled(queue)
-                    for msg in running:
-                        self._helper.push.unhandled(queue, msg)
-                    for msg in pending + running:
-                        self._helper.status_update(
-                            msg.execution_id, "Finished", error=f"Procedure worker failed: {error}"
-                        )
-                    self._helper.notify_watchers(queue, "unhandled")
-                self._helper.remove_from_active.by_queue(queue)
-                self._run_callbacks(queue)
-                self._helper.notify_watchers(queue, "execution")
-                del self._active_workers[queue]
-
-        return cleanup_worker
+    def _procedure_request_callback(self, msg: MessageObject):
+        """Validate the personal topic against the submitted ACL identity before enqueueing."""
+        message = msg.value
+        if not isinstance(message, ProcedureRequestMessage):
+            logger.warning("Received an invalid procedure request.")
+            return
+        client_info = message.metadata.get("client_info")
+        token = client_info.pop("acl_token", None) if isinstance(client_info, dict) else None
+        if token is not None and not isinstance(token, str):
+            self._ack(False, "Invalid procedure ACL token type.", message.execution_id)
+            return
+        username_regex = MessageEndpoints.procedure_request("([^/]+)").endpoint
+        match = re.fullmatch(username_regex, msg.topic)
+        if not match:
+            self._ack(False, "Could not extract username from topic.", message.execution_id)
+            return
+        if not isinstance(client_info, dict) or not client_info:
+            self._ack(False, "Missing client info in request metadata.", message.execution_id)
+            return
+        acl_user = client_info.get("acl_user") or "default"
+        if acl_user != match.group(1):
+            self._ack(False, "Username in topic does not match client info.", message.execution_id)
+            return
+        return self._process_queue_request(msg, token=token)
 
     def _respond_to_valid_request(self, message: ProcedureRequestMessage):
-        self._ack(True, f"Running procedure {message.identifier}", message.execution_id)
         queue = message.queue or PROCEDURE.WORKER.DEFAULT_QUEUE
         exec_message = ProcedureExecutionMessage(
             identifier=message.identifier,
@@ -333,16 +338,9 @@ class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureEx
             args_kwargs=message.args_kwargs or ((), {}),
             execution_id=message.execution_id,
         )
-        logger.debug(f"active workers: {self._active_workers}, worker requested: {queue}")
-        self._helper.push.exec(queue, exec_message)
+        logger.debug(f"Procedure submitted to logical queue {queue}")
         return exec_message
 
-    def _process_queue_request(self, msg: MessageObject):
-        with self.lock:
-            if (exec_msg := super()._process_queue_request(msg)) is not None:
-                self._messages_by_ids[exec_msg.execution_id] = exec_msg
-        return exec_msg
-
     def _publish_metrics(self, msg: ProcedureQNotifMessage):
         self._conn.publish_metrics(
             "pending_procedure_queues",
@@ -385,65 +383,6 @@ class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureEx
             self._callbacks[queue] = []
         self._callbacks[queue].append(cb)
 
-    def _run_callbacks(self, queue: str):
-        with self.lock:
-            if queue not in self._active_workers:
-                logger.error(f"Attempted to run callbacks for nonexistent worker {queue}")
-                return
-            if (worker := self._active_workers[queue]["worker"]) is None:
-                return
-        for cb in self._callbacks.get(queue, []):
-            cb(worker)
-        self._callbacks[queue] = []
-
-    def _process_abort(self, msg: MessageObject):
-        message: ProcedureAbortMessage = msg.value  # type: ignore
-        with self.lock:
-            if message.abort_all:
-                self._abort_all()
-            if message.queue is not None:
-                self._abort_queue(message.queue)
-            if message.execution_id is not None:
-                self._abort_execution(message.execution_id)
-
-    def _abort_execution(self, execution_id: str):
-        if (msg := self._messages_by_ids.get(execution_id)) is None:
-            logger.warning(f"Procedure execution with ID {execution_id} not known.")
-            return
-        # Remove it from the queue if not yet started
-        if self._conn.lrem(MessageEndpoints.procedure_execution(msg.queue), 0, msg) > 0:
-            logger.debug(f"Removed execution {msg} from queue.")
-            self._helper.notify_watchers(msg.queue, "execution")
-        # Otherwise try to remove it from whichever worker has it
-        for entry in self._active_workers.values():
-            if (worker := entry["worker"]) is not None and worker:
-                worker.abort_execution(execution_id)
-        # Move it to unhandled and stop tracking
-        self._helper.push.unhandled(msg.queue, msg)
-        del self._messages_by_ids[execution_id]
-
-    def _abort_queue(self, queue: str):
-        self._helper.move.execution_queue_to_unhandled(queue)
-        if (entry := self._active_workers.get(queue)) is not None:
-            if entry["worker"] is not None:
-                entry["worker"].abort()
-            entry["future"].cancel()
-            futures.wait((entry["future"],), PROCEDURE.MANAGER_SHUTDOWN_TIMEOUT_S)
-        else:
-            logger.warning(f"Received abort request for unknown queue {queue}!")
-        self._helper.notify_watchers(queue, "execution")
-
-    def _abort_all(self):
-        with self.lock:
-            entries = list(self._active_workers.values())
-            for entry in entries:
-                if entry["worker"] is not None:
-                    entry["worker"].abort()
-            for entry in entries:
-                entry["future"].cancel()
-        self._helper.move.all_execution_queues_to_unhandled()
-        self._wait_for_all_futures()
-
     def _process_clear_unhandled(self, msg: MessageObject):
         message: ProcedureClearUnhandledMessage = msg.value  # type: ignore
         with self.lock:
@@ -458,4 +397,5 @@ class ProcedureManager(ProcedureManagerBase[ProcedureRequestMessage, ProcedureEx
         """Remove any callbacks from any Redis endpoints"""
         self._conn.unregister(self._abort_ep, None, self._process_abort)
         self._conn.unregister(self._clear_uh_ep, None, self._process_clear_unhandled)
-        self._conn.unregister(self._request_ep, None, self._process_queue_request)
+        self._conn.unregister(patterns=self._request_ep, cb=self._procedure_request_callback)
+        self._conn.unregister(self._q_notif_ep, None, self._publish_metrics)