bec-server 4.3.0__tar.gz → 4.3.2__tar.gz

{bec_server-4.3.0 → bec_server-4.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: bec-server
-Version: 4.3.0
+Version: 4.3.2
 Summary: BEC server
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

{bec_server-4.3.0 → bec_server-4.3.2}/bec_server/actors/worker.py

@@ -1,14 +1,12 @@
 import importlib
 import os
-from contextlib import redirect_stdout
 from inspect import isclass
 
 from bec_lib.client import BECClient
-from bec_lib.logger import LogLevel, bec_logger
+from bec_lib.logger import bec_logger
 from bec_lib.messages import ProcedureWorkerStatus
-from bec_lib.redis_connector import RedisConnector
 from bec_server.actors.actor import ActorBase
-from bec_server.procedures.oop_worker_base import RedisOutputDiverter, get_env, push_status, setup
+from bec_server.procedures.oop_worker_base import get_env, push_status, setup
 from bec_server.procedures.subprocess_worker import SubProcessWorker
 
 logger = bec_logger.logger
@@ -53,20 +51,12 @@ def get_actor_env():
 def main():
     env, helper, client, conn = setup(get_env())
     actor_env = get_actor_env()
-    logger_connector = RedisConnector(env["redis_server"])
-    output_diverter = RedisOutputDiverter(logger_connector, env["queue"])
-    with redirect_stdout(output_diverter):
-        logger.add(
-            output_diverter,
-            level=LogLevel.SUCCESS,
-            format=bec_logger.formatting(is_container=True),
-            filter=bec_logger.filter(),
-        )
-        logger.success(f"Starting ActorProcedureWorker with env: {actor_env}")
+    # The parent SubProcessWorker streams stdout and stderr to Redis.
+    logger.success(f"Starting ActorProcedureWorker with env: {actor_env}")
     push_status(conn, env["queue"], ProcedureWorkerStatus.IDLE)
     actor_procedure(bec=client, **actor_env)
+    push_status(conn, env["queue"], ProcedureWorkerStatus.FINISHED)
     conn.shutdown()
-    logger_connector.shutdown()
 
 
 if __name__ == "__main__":

{bec_server-4.3.0 → bec_server-4.3.2}/bec_server/procedures/constants.py

@@ -18,6 +18,7 @@ class OopWorkerEnv(TypedDict):
     queue: str
     timeout_s: str
     client_class: NotRequired[BecClientType]
+    worker_id: NotRequired[str]
 
 
 P = ParamSpec("P")
@@ -33,7 +34,25 @@ class BecProcedure(Protocol[P]):
     def __call__(self, *args: P.args, **kwargs: P.kwargs) -> None: ...
 
 
-class ProcedureWorkerError(RuntimeError): ...
+class ProcedureWorkerFailure(str, Enum):
+    """Fixed, credential-free descriptions suitable for public execution status."""
+
+    STARTUP = "Procedure worker startup failed"
+    AUTHENTICATION = "Procedure worker authentication failed"
+    EXITED = "Procedure worker exited unexpectedly"
+    INPUT_CLOSED = "Procedure worker input closed unexpectedly"
+    ABORTED = "Procedure worker was aborted"
+
+
+class ProcedureWorkerError(RuntimeError):
+    """Worker failure with an optional safe category for public status."""
+
+    def __init__(self, *args: object, failure: ProcedureWorkerFailure | None = None) -> None:
+        """Default to the public failure description unless internal details are supplied."""
+        if not args and failure is not None:
+            args = (failure.value,)
+        super().__init__(*args)
+        self.failure = failure
 
 
 class WorkerAlreadyExists(ProcedureWorkerError): ...
@@ -48,7 +67,7 @@ class NoImage(ProcedureWorkerError): ...
 @dataclass(frozen=True)
 class _WORKER:
     MAX_WORKERS = 10
-    QUEUE_TIMEOUT_S = 10
+    QUEUE_TIMEOUT_S = 60
     DEFAULT_QUEUE = "primary"
 
 

{bec_server-4.3.0 → bec_server-4.3.2}/bec_server/procedures/container_utils.py

@@ -10,7 +10,7 @@ from typing import Iterator, Literal, cast
 
 from podman import PodmanClient
 from podman.domain.containers import Container
-from podman.errors import APIError
+from podman.errors import APIError, NotFound
 
 from bec_lib.logger import bec_logger
 from bec_server.procedures.constants import (
@@ -81,6 +81,67 @@ class _PodmanUtilsBase(ContainerCommandBackend):
             tag=f"{PROCEDURE.CONTAINER.IMAGE_NAME}:v{PROCEDURE.BEC_VERSION}",
         )
 
+    def run_interactive(
+        self,
+        image_tag: str,
+        environment: OopWorkerEnv,
+        volumes: list[VolumeSpec],
+        command: str,
+        pod_name: str | None = None,
+        container_name: str | None = None,
+    ) -> subprocess.Popen:
+        """Attach process pipes so private worker bootstrap never enters Podman metadata.
+
+        Args:
+            image_tag (str): Worker image.
+            environment (OopWorkerEnv): Non-secret worker settings.
+            volumes (list[VolumeSpec]): Code mounts.
+            command (str): Worker entry point.
+            pod_name (str | None): Existing Podman pod.
+            container_name (str | None): Unique worker container name.
+        Returns:
+            subprocess.Popen: Attached process with stdin and separate output pipes.
+        """
+        prefix = self._cli_prefix()
+        args = [*prefix, "create", "-i"]
+        args.extend(_multi_args_from_dict("-e", environment))
+        for volume in volumes:
+            args.extend(
+                [
+                    "-v",
+                    f"{volume['source']}:{volume['target']}{':ro' if volume['read_only'] else ''}",
+                ]
+            )
+        if pod_name:
+            args.extend(["--pod", pod_name])
+        if container_name:
+            args.extend(["--name", container_name])
+        # Create synchronously: cancelling the attached start cannot leave a later
+        # `podman run` creating an orphan container after cleanup has returned.
+        container_id = _run_and_capture_error(*args, image_tag, command).stdout.decode().strip()
+        try:
+            return subprocess.Popen(
+                [*prefix, "start", "--attach", "--interactive", container_id],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                bufsize=0,
+            )
+        except BaseException:
+            self.remove(container_id)
+            raise
+
+    def _cli_prefix(self) -> list[str]:
+        return ["podman", "--url", self.uri] if isinstance(self, PodmanApiUtils) else ["podman"]
+
+    def remove(self, id: str) -> None:
+        """Remove a stopped or starting worker container, preventing a later start.
+
+        Args:
+            id (str): Container name or ID.
+        """
+        _run_and_capture_error(*self._cli_prefix(), "rm", "--force", "--ignore", id)
+
 
 class PodmanApiOutput(ContainerCommandOutput):
     def __init__(self, command_output: Iterator[bytes]):
@@ -157,7 +218,10 @@ class PodmanApiUtils(_PodmanUtilsBase):
 
     def state(self, id: str) -> PodmanContainerStates | None:
         with PodmanClient(base_url=self.uri) as client:
-            status = client.containers.get(id).status
+            try:
+                status = client.containers.get(id).status
+            except NotFound:
+                return None
             if status == "unknown":
                 return None
             return PodmanContainerStates(status)
@@ -220,7 +284,7 @@ class PodmanCliUtils(_PodmanUtilsBase):
         _name_arg = ["--replace", "--name", container_name] if container_name else []
         if container_name:
             # in case of incomplete cleanup previously, even with --replace it is possible to get an error
-            _run_and_capture_error("podman", "rm", "-f", container_name)
+            _run_and_capture_error("podman", "rm", "-f", "--ignore", container_name)
         return (
             _run_and_capture_error(
                 "podman",
@@ -251,14 +315,19 @@ class PodmanCliUtils(_PodmanUtilsBase):
 
     def logs(self, id: str) -> list[str]:
         try:
-            return _run_and_capture_error("podman", "logs", id).stderr.decode().splitlines()
+            output = _run_and_capture_error("podman", "logs", id)
+            return (output.stdout + output.stderr).decode(errors="replace").splitlines()
         except ProcedureWorkerError as e:
             logger.error(e)
             return [f"No logs found for container {id}\n"]
 
     def state(self, id: str) -> PodmanContainerStates | None:
         for container in self._podman_ls_json():
-            if container["Id"] == id or container["Id"].startswith(id):
+            if (
+                container["Id"] == id
+                or container["Id"].startswith(id)
+                or id in container.get("Names", [])
+            ):
                 if names := container.get("Names"):
                     logger.debug(f"Container {names[0]} status: {container['State']}")
                 return PodmanContainerStates(container["State"])

bec_server-4.3.2/bec_server/procedures/container_worker.py

@@ -0,0 +1,152 @@
+import time
+
+from bec_lib.logger import bec_logger
+from bec_lib.redis_connector import RedisConnector
+from bec_server.procedures.constants import PROCEDURE, PodmanContainerStates
+from bec_server.procedures.container_utils import get_backend
+from bec_server.procedures.oop_worker_base import (
+    main,  # temporarily needed before update of base image on release
+)
+from bec_server.procedures.oop_worker_base import PROCESS_TIMEOUT, OutOfProcessWorkerBase
+from bec_server.procedures.protocol import ContainerCommandBackend
+
+logger = bec_logger.logger
+
+
+class ContainerProcedureWorker(OutOfProcessWorkerBase):
+    """A worker which runs scripts in a container with a full BEC environment,
+    mounted from the filesystem, and only access to Redis"""
+
+    # The Podman client is a thin wrapper around the libpod API
+    # documented at https://docs.podman.io/en/latest/_static/api.html
+    # which is more detailed than the podman-py documentation
+
+    def _setup_execution_environment(self):
+        self._backend: ContainerCommandBackend = get_backend()
+        image_tag = f"{PROCEDURE.CONTAINER.IMAGE_NAME}:v{PROCEDURE.BEC_VERSION}"
+        self.container_name = (
+            f"bec_procedure_{PROCEDURE.BEC_VERSION}_{self._worker_id or self._queue}"
+        )
+        if not self._backend.image_exists(image_tag):
+            self._backend.build_worker_image()
+        environment = self._worker_environment() | self._additional_env
+        if self._worker_id is not None:
+            self._container_id = self.container_name
+            self._process = self._backend.run_interactive(
+                image_tag,
+                environment,
+                [
+                    {
+                        "source": str(PROCEDURE.CONTAINER.DEPLOYMENT_PATH),
+                        "target": "/bec",
+                        "type": "bind",
+                        "read_only": True,
+                    }
+                ],
+                PROCEDURE.CONTAINER.COMMAND,
+                pod_name=PROCEDURE.CONTAINER.POD_NAME,
+                container_name=self.container_name,
+            )
+            output_conn = RedisConnector([self._redis_server], name="Procedure container output")
+            try:
+                if self._control_credentials is not None:
+                    output_conn.authenticate(
+                        username=self._control_credentials["username"] or "default",
+                        password=self._control_credentials["token"],
+                    )
+                self._start_output_capture(output_conn)
+            except BaseException:
+                output_conn.shutdown()
+                self._kill_process()
+                raise
+            return
+        self._container_id = self._backend.run(
+            image_tag,
+            environment,
+            [
+                {
+                    "source": str(PROCEDURE.CONTAINER.DEPLOYMENT_PATH),
+                    "target": "/bec",
+                    "type": "bind",
+                    "read_only": True,
+                }
+            ],
+            PROCEDURE.CONTAINER.COMMAND,
+            pod_name=PROCEDURE.CONTAINER.POD_NAME,
+            container_name=self.container_name,
+        )
+
+    def _ending_or_ended(self):
+        if self._worker_id is not None:
+            return self._process.poll() is not None
+        return self._backend.state(self._container_id) in [
+            PodmanContainerStates.EXITED,
+            PodmanContainerStates.STOPPED,
+            PodmanContainerStates.STOPPING,
+        ]
+
+    def _kill_process(self):
+        if self._worker_id is not None:
+            # The attached CLI can fail while the container is still running.
+            terminal = (
+                None,
+                PodmanContainerStates.CREATED,
+                PodmanContainerStates.EXITED,
+                PodmanContainerStates.STOPPED,
+            )
+            if self._backend.state(self.container_name) not in terminal:
+                self._backend.interrupt(self.container_name)
+                deadline = time.monotonic() + PROCESS_TIMEOUT
+                while time.monotonic() < deadline:
+                    if self._backend.state(self.container_name) in terminal:
+                        break
+                    time.sleep(0.1)
+                else:
+                    self._backend.kill(self.container_name)
+                deadline = time.monotonic() + PROCESS_TIMEOUT
+                while self._backend.state(self.container_name) not in terminal:
+                    if time.monotonic() >= deadline:
+                        raise RuntimeError("Procedure container failed to stop")
+                    time.sleep(0.1)
+            # Creation completed before the launcher was started, so removal also
+            # prevents an attached start that has not reached the engine yet.
+            self._backend.remove(self.container_name)
+            if self._process.stdin is not None:
+                self._process.stdin.close()
+            try:
+                self._process.wait(PROCESS_TIMEOUT)
+            except Exception:
+                self._process.kill()
+                self._process.wait(PROCESS_TIMEOUT)
+            return
+        if not self._ending_or_ended():
+            self._backend.interrupt(self.container_name)
+            start = time.monotonic()
+            while time.monotonic() < start + PROCESS_TIMEOUT:
+                if self._ending_or_ended():
+                    return
+                time.sleep(0.2)
+            logger.warning(
+                f"Procedure worker {self._container_id} for queue {self._queue} failed to shut down, killing."
+            )
+            self._backend.kill(self.container_name)
+
+    def abort_execution(self, execution_id: str):
+        """Abort the execution with the given id. Has no effect if the given ID is not the current job"""
+        if execution_id == self._current_execution_id:
+            self._backend.kill(self._container_id)
+            self._helper.remove_from_active.by_exec_id(execution_id)
+            logger.info(
+                f"Aborting execution {execution_id}, restarting worker for queue: {self._queue}"
+            )
+            self._setup_execution_environment()
+
+    def logs(self):
+        if self._worker_id is not None:
+            if self._process.poll() is not None:
+                self._join_output()
+            with self._output_lock:
+                return "".join(self._output_logs).splitlines()
+        if self._container_id is None:
+            return [""]
+        return self._backend.logs(self._container_id)