manta-node 0.5b0__py3-none-any.whl

manta_node/infrastructure/container/__init__.py

@@ -0,0 +1,11 @@
+"""Container infrastructure - Docker runtime management."""
+
+from .docker_adapter import DockerRaiser
+from .image_executor import ImageExecutor
+from .manager import ContainerManager
+
+__all__ = [
+    "ContainerManager",
+    "DockerRaiser",
+    "ImageExecutor",
+]

manta_node/infrastructure/container/docker_adapter.py

@@ -0,0 +1,253 @@
+from datetime import datetime
+from typing import Optional
+
+from aiodocker import Docker
+from aiodocker.docker import DockerContainer
+from docker import DockerClient
+
+from manta_common.errors import MantaDockerError
+from manta_common.traces import Tracer
+
+
+class DockerRaiser:
+    async_client: Docker
+    tracer: Tracer
+    docker_client: DockerClient
+
+    async def run_or_raise(self, config: dict) -> DockerContainer:
+        """
+        Start, pull or run a container
+
+        Parameters
+        ----------
+        config : dict
+            Configuration
+
+        Returns
+        -------
+        DockerContainer
+            Container
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot run container
+        """
+        container = None
+        try:
+            container = await self.async_client.containers.run(config)
+        except Exception as exc:
+            self.tracer.error("Task cannot be started")
+            raise MantaDockerError(
+                "Task cannot be started", metadata=self.tracer.metadata
+            ) from exc
+        return container
+
+    async def pull_image_or_raise(self, image: str):
+        """
+        Pull an image
+
+        Parameters
+        ----------
+        image : str
+            Image to pull
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot pull image
+        """
+        try:
+            await self.async_client.images.pull(image)
+        except Exception as exc:
+            self.tracer.error(f"Image {image} cannot be pulled")
+            raise MantaDockerError(
+                f"Image {image} cannot be pulled", metadata=self.tracer.metadata
+            ) from exc
+
+    async def list_containers_or_raise(self, all_containers: bool = True) -> list:
+        """
+        List all containers in the docker engine
+
+        Parameters
+        ----------
+        all_containers : bool, optional
+            Show all containers; only running containers are shown by default
+
+        Returns
+        -------
+        list
+            List of containers
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot list containers
+        """
+        containers = []
+        try:
+            containers = await self.async_client.containers.list(all=all_containers)
+        except Exception as exc:
+            self.tracer.error("Unable to list containers")
+            raise MantaDockerError(
+                "Unable to list containers", metadata=self.tracer.metadata
+            ) from exc
+        return containers
+
+    def list_images_or_raise(self) -> list:
+        """
+        List all images in the docker engine
+
+        Returns
+        -------
+        list
+            List of images
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot list images
+        """
+        images = []
+        try:
+            images = [
+                tag for image in self.docker_client.images.list() for tag in image.tags
+            ]
+        except Exception as exc:
+            self.tracer.error("Unable to list images")
+            raise MantaDockerError(
+                "Unable to list images", metadata=self.tracer.metadata
+            ) from exc
+        return images
+
+    async def restart_or_raise(self, container: DockerContainer):
+        """
+        Restart container
+
+        Parameters
+        ----------
+        container : DockerContainer
+            Container
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot restart container
+        """
+        try:
+            await container.restart()
+        except Exception as exc:
+            self.tracer.error("Container cannot be restarted")
+            raise MantaDockerError(
+                "Container cannot be restart", metadata=self.tracer.metadata
+            ) from exc
+
+    async def stop_or_raise(self, container: DockerContainer):
+        """
+        Stop container
+
+        Parameters
+        ----------
+        container : DockerContainer
+            Container
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot stop container
+        """
+        try:
+            await container.stop()
+        except Exception as exc:
+            self.tracer.error("Container cannot be stopped")
+            raise MantaDockerError(
+                "Container cannot be stopped", metadata=self.tracer.metadata
+            ) from exc
+
+    async def remove_or_raise(self, container: DockerContainer):
+        """
+        Remove container
+
+        Parameters
+        ----------
+        container : DockerContainer
+            Container
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot remove container
+        """
+        try:
+            await container.delete(force=True)
+        except Exception as exc:
+            self.tracer.error("Container cannot be removed")
+            raise MantaDockerError(
+                "Container cannot be removed", metadata=self.tracer.metadata
+            ) from exc
+
+    async def get_status_or_raise(self, container: DockerContainer):
+        """
+        Reload container
+
+        Parameters
+        ----------
+        container : DockerContainer
+            Container
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot reload container
+        """
+        status = "unknown"
+        try:
+            status = await container.show()
+            status = status["State"]["Status"]
+        except Exception as exc:
+            self.tracer.error("Cannot get status of container")
+            raise MantaDockerError(
+                "Cannot get status of container", metadata=self.tracer.metadata
+            ) from exc
+        return status
+
+    async def get_container_logs_or_raise(
+        self, container: DockerContainer, since: Optional[datetime] = None
+    ) -> bytes:
+        """
+        Get container logs (async)
+
+        Parameters
+        ----------
+        container : DockerContainer
+            Container
+        since: Optional[datetime]
+            Start time
+
+        Returns
+        -------
+        bytes
+            Logs
+
+        Raises
+        ------
+        MantaDockerError
+            Cannot get logs
+        """
+        logs = b""
+        try:
+            # Use async Docker client to avoid blocking the event loop
+            async_container = await self.async_client.containers.get(container.id)
+            # aiodocker >= 0.25: log() returns a coroutine yielding a list of strings
+            # since must be a unix timestamp (int), not a datetime object
+            since_ts = int(since.timestamp()) if since else None
+            log_lines = await async_container.log(
+                stdout=True, stderr=True, since=since_ts
+            )
+            logs = "".join(log_lines).encode()
+        except Exception as exc:
+            self.tracer.error(f"Cannot get logs of container: {exc}")
+            raise MantaDockerError(
+                "Cannot get logs of container", metadata=self.tracer.metadata
+            ) from exc
+        return logs

manta_node/infrastructure/container/image_executor.py

@@ -0,0 +1,222 @@
+import asyncio
+from datetime import datetime, timezone
+from typing import Dict, Optional, Tuple
+
+import psutil
+
+from aiodocker.docker import DockerContainer
+
+from manta_common.build.common.tasks import TaskStatus
+from manta_common.conversions import ID
+from manta_common.errors import MantaDockerError
+from manta_common.traces import Tracer
+
+from ...domain.task_lifecycle import TaskLifecycleService
+from .docker_adapter import DockerRaiser
+
+ORPHAN_PORT = 8080
+
+
+class ImageExecutor(DockerRaiser):
+    """
+    Runs plain Docker images with native ENTRYPOINT/CMD.
+    No zipapp, no manta SDK, no gRPC inside the container.
+    """
+
+    tracer: Tracer
+    containers: Dict[Tuple[ID, ID], DockerContainer]
+
+    @staticmethod
+    def _is_manta_orphan(proc: psutil.Process) -> bool:
+        """Return True if *proc* looks like a manta worker that escaped
+        its ``--network=host`` container.
+
+        Checks that the process (a) runs a manta worker cmdline and
+        (b) has ``TASK_ID`` / ``SWARM_ID`` in its environment — these
+        env vars are injected by ``run_image`` and are unique to manta
+        containers.  This avoids false positives from non-manta
+        uvicorn processes or other teams' containers.
+        """
+        try:
+            cmd = " ".join(proc.cmdline()).lower()
+            if "worker_main" not in cmd and "uvicorn" not in cmd:
+                return False
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            return False
+
+        try:
+            env = proc.environ()
+            if "TASK_ID" not in env or "SWARM_ID" not in env:
+                return False
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return False
+
+        return True
+
+    async def _ensure_port_free(self, port: int, tracked: set) -> None:
+        """Kill manta-orphan processes holding ``port``.
+
+        When a ``--network=host`` container is killed its child process
+        can escape the PID namespace and remain on the host, still holding
+        the port.  Only processes that match the manta worker profile
+        (``worker_main``/``uvicorn`` cmdline AND ``TASK_ID``/``SWARM_ID``
+        in environment) AND are NOT in *tracked* are targeted —
+        co-located host services, other teams' containers, and live
+        tracked workers are never touched.
+        """
+        try:
+            conns = list(psutil.net_connections(kind="inet"))
+        except psutil.AccessDenied:
+            self.tracer.warning(
+                "Cannot enumerate network connections (non-root?); "
+                "skipping port cleanup"
+            )
+            return
+        for conn in conns:
+            if conn.status != "LISTEN" or conn.laddr.port != port:
+                continue
+            if conn.pid is None:
+                continue
+            try:
+                proc = psutil.Process(conn.pid)
+                if not self._is_manta_orphan(proc):
+                    continue
+                # Double-check: don't kill a live tracked worker
+                try:
+                    env = proc.environ()
+                    task_xid = env.get("TASK_ID")
+                    swarm_xid = env.get("SWARM_ID")
+                    if task_xid and swarm_xid:
+                        tid_pair = (ID(swarm_xid).tid, ID(task_xid).tid)
+                        if tid_pair in tracked:
+                            continue
+                except (psutil.NoSuchProcess, psutil.AccessDenied):
+                    pass
+                self.tracer.warning(
+                    f"Manta orphan process {conn.pid} ({proc.name()}) "
+                    f"holding port {port}, sending SIGTERM"
+                )
+                proc.terminate()
+                gone, _ = await asyncio.to_thread(psutil.wait_procs, [proc], timeout=5)
+                if not gone:
+                    self.tracer.warning(
+                        f"Process {conn.pid} did not exit after SIGTERM, "
+                        f"sending SIGKILL"
+                    )
+                    proc.kill()
+                    await asyncio.to_thread(psutil.wait_procs, [proc], timeout=3)
+            except (
+                psutil.NoSuchProcess,
+                psutil.AccessDenied,
+                psutil.ZombieProcess,
+            ):
+                pass
+
+    async def run_image(
+        self,
+        swarm_id: ID,
+        task_id: ID,
+        image: str,
+        environment: Optional[dict] = None,
+    ) -> datetime:
+        """
+        Pull (if not present locally) and start a Docker image with its native ENTRYPOINT.
+
+        Returns
+        -------
+        datetime
+            Container start time
+        """
+        tracked = set(getattr(self, "tasks", None) and self.tasks._env_ids or [])
+        tracked |= getattr(self, "_launching", None) or set()
+        await self._ensure_port_free(ORPHAN_PORT, tracked)
+        try:
+            await self.async_client.images.inspect(image)
+        except Exception:
+            self.tracer.info(f"Image {image} not found locally, pulling...")
+            await self.pull_image_or_raise(image)
+
+        env = [f"{k}={v}" for k, v in environment.items()] if environment else None
+        config = {
+            "Image": image,
+            "HostConfig": {
+                "NetworkMode": "host",
+                "ExtraHosts": ["host.docker.internal:host-gateway"],
+            },
+            "Env": env,
+            "Detach": True,
+            "LogConfig": {
+                "type": "json-file",
+                "config": {"max-size": "10m", "max-file": "3"},
+            },
+            "Labels": {
+                "com.docker.compose.project": "manta_tasks",
+                "manta.swarm-id": ID(swarm_id).xid,
+                "manta.task-id": ID(task_id).xid,
+            },
+        }
+
+        start_time = datetime.now(timezone.utc)
+        self.tracer.debug(f"Starting image {image} for task {task_id}")
+        self.containers[(swarm_id, task_id)] = await self.run_or_raise(config)
+        self.tracer.info(f"Task {task_id} image container started.")
+        return start_time
+
+    async def get_image_task_status(self, swarm_id: ID, task_id: ID) -> TaskStatus:
+        """
+        Exit-code-aware status for image-mode tasks (no SDK inside the container).
+        """
+        container = self.containers.get((swarm_id, task_id))
+        if container is None:
+            self.tracer.info(
+                f"ImageExecutor.get_image_task_status: container NOT FOUND "
+                f"task={task_id} swarm={swarm_id} "
+                f"containers_keys={list(self.containers.keys())}"
+            )
+            raise MantaDockerError("Container not found", metadata=self.tracer.metadata)
+
+        self.tracer.info(
+            f"ImageExecutor.get_image_task_status: container FOUND "
+            f"task={task_id} swarm={swarm_id} container_id={container.id}"
+        )
+        inspect = await container.show()
+        container_status = inspect["State"]["Status"]
+        exit_code = inspect["State"].get("ExitCode", 0)
+        self.tracer.info(
+            f"ImageExecutor.get_image_task_status: "
+            f"container_status={container_status} exit_code={exit_code} "
+            f"task={task_id}"
+        )
+
+        status = TaskLifecycleService.status_from_exit(container_status, exit_code)
+        self.tracer.info(
+            f"ImageExecutor.get_image_task_status: "
+            f"container_status={container_status} exit_code={exit_code} "
+            f"→ TaskStatus={status} task={task_id}"
+        )
+        return status
+
+    async def stop_image(
+        self,
+        swarm_id: ID,
+        task_id: ID,
+        timeout: Optional[int] = 10,
+    ) -> None:
+        """
+        Stop a running image-mode container.
+
+        Sends SIGTERM and waits up to ``timeout`` seconds for the container to
+        exit, then falls back to SIGKILL (handled by the Docker daemon).
+        Idempotent: aiodocker treats an already-stopped container as success.
+
+        The container reference is retained in ``self.containers`` so that
+        ``get_image_task_status`` can still return the final exit status.
+        Removal is the caller's responsibility (see ``remove_container``).
+        """
+        container = self.containers.get((swarm_id, task_id))
+        if container is None:
+            raise MantaDockerError("Container not found", metadata=self.tracer.metadata)
+
+        self.tracer.debug(f"Stopping container for task {task_id}")
+        await container.stop(timeout=timeout)
+        self.tracer.info(f"Task {task_id} container stopped.")