manta-node 0.5b0.dev259__py3-none-any.whl → 0.5b0.dev264__py3-none-any.whl

manta_node/infrastructure/container/docker_adapter.py

@@ -1,3 +1,5 @@
+import asyncio
+import concurrent.futures
 from datetime import datetime
 
 from aiodocker import Docker
@@ -13,6 +15,80 @@ class DockerRaiser:
     tracer: Tracer
     docker_client: DockerClient
 
+    # Dedicated, bounded executor for image pulls only. A multi-GB pull holds
+    # a thread for the full download; routing pulls through their own small
+    # pool (instead of asyncio.to_thread's shared default ThreadPoolExecutor,
+    # min(32, cpu+4) workers) keeps several concurrent large pulls from
+    # starving unrelated to_thread/run_in_executor work on the node. Process-
+    # lifetime singleton — the node process runs indefinitely, so no explicit
+    # shutdown is needed. image_needs_pull's inspect_distribution check is
+    # NOT routed through this pool: it's quick and issued right before the
+    # pull, and sharing a 4-slot pool with in-flight pulls could block it
+    # waiting for a pull slot.
+    _PULL_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
+        max_workers=4, thread_name_prefix="docker-pull"
+    )
+
+    async def image_needs_pull(self, image: str) -> bool:
+        """
+        Check whether ``image`` is missing locally or stale relative to the registry.
+
+        Compares the local image's ``RepoDigests`` against the remote manifest
+        digest, obtained via the Docker daemon's own distribution API
+        (``docker_client.api.inspect_distribution``). This reuses the daemon's
+        registry auth, media-type negotiation and multi-arch resolution — the
+        same machinery ``docker pull`` itself relies on — instead of a bespoke
+        registry client.
+
+        Parameters
+        ----------
+        image : str
+            Image reference to check (e.g. ``"repo/image:latest"``)
+
+        Returns
+        -------
+        bool
+            True if the image should be (re)pulled.
+
+        Notes
+        -----
+        Graceful degradation: if the registry check itself fails (unreachable,
+        unauthenticated, rate-limited, ...), a warning is logged and this falls
+        back to "pull only if absent locally" — it never raises out of task
+        launch.
+        """
+        try:
+            local = await self.async_client.images.inspect(image)
+        except Exception:
+            # Image not present locally: must pull regardless of the registry.
+            return True
+
+        local_digests = {
+            digest.split("@", 1)[1]
+            for digest in (local.get("RepoDigests") or [])
+            if "@" in digest
+        }
+        if not local_digests:
+            # Locally-built/tagged image (never pushed/pulled), or only
+            # malformed RepoDigests entries: nothing to compare against the
+            # registry. Treat as up-to-date rather than overwriting a
+            # local-only image on every launch.
+            return False
+
+        try:
+            distribution = await asyncio.to_thread(
+                self.docker_client.api.inspect_distribution, image
+            )
+            remote_digest = distribution["Descriptor"]["digest"]
+        except Exception as exc:
+            self.tracer.warning(
+                f"Cannot verify registry digest for {image}, skipping "
+                f"auto-pull staleness check: {exc!r}"
+            )
+            return False
+
+        return remote_digest not in local_digests
+
     async def run_or_raise(self, config: dict) -> DockerContainer:
         """
         Start, pull or run a container
@@ -74,6 +150,19 @@ class DockerRaiser:
         """
         Pull an image
 
+        Pulls via the sync docker-py client rather than the async aiodocker
+        client. docker-py auto-resolves registry credentials from
+        ``~/.docker/config.json`` (including credHelpers/credsStore) — the
+        same client ``image_needs_pull`` already uses for
+        ``inspect_distribution`` — whereas aiodocker pulls anonymously and
+        404s on private images with "pull access denied ... may require
+        'docker login'" (refs #348).
+
+        Routed through ``self._PULL_EXECUTOR`` (a dedicated, bounded thread
+        pool) rather than ``asyncio.to_thread``'s shared default executor, so
+        a long-running multi-GB pull can't starve unrelated to_thread/
+        run_in_executor work elsewhere on the node (refs #347 review).
+
         Parameters
         ----------
         image : str
@@ -85,7 +174,10 @@ class DockerRaiser:
             Cannot pull image
         """
         try:
-            await self.async_client.images.pull(image)
+            loop = asyncio.get_running_loop()
+            await loop.run_in_executor(
+                self._PULL_EXECUTOR, self.docker_client.images.pull, image
+            )
         except Exception as exc:
             self.tracer.error(f"Image {image} cannot be pulled")
             raise MantaDockerError(

manta_node/infrastructure/container/image_executor.py

@@ -115,37 +115,80 @@ class ImageExecutor(DockerRaiser):
         task_id: ID,
         image: str,
         environment: dict | None = None,
+        gpu: bool = False,
+        force_pull: bool = False,
     ) -> datetime:
         """
-        Pull (if not present locally) and start a Docker image with its native ENTRYPOINT.
+        Pull (when missing or stale) and start a Docker image with its native ENTRYPOINT.
+
+        Parameters
+        ----------
+        swarm_id : ID
+            Swarm ID
+        task_id : ID
+            Task ID
+        image : str
+            Image to run
+        environment : Optional[dict]
+            Environment variables to inject into the container
+        gpu : bool
+            Attach GPU support, reusing the same ``gpu_config`` detection as
+            ``ContainerManager.run_container``
+        force_pull : bool
+            Force a pull before running, bypassing the digest staleness check
 
         Returns
         -------
         datetime
             Container start time
+
+        Raises
+        ------
+        MantaDockerError
+            GPU requested but not supported by this container manager
         """
         tracked = set(getattr(self, "tasks", None) and self.tasks._env_ids or [])
         tracked |= getattr(self, "_launching", None) or set()
         await self._ensure_port_free(ORPHAN_PORT, tracked)
-        try:
-            await self.async_client.images.inspect(image)
-        except Exception:
-            self.tracer.info(f"Image {image} not found locally, pulling...")
+
+        # gpu_config is provided by ContainerManager via MRO in the real
+        # TaskManager(ContainerManager, ImageExecutor) composition; guard with
+        # getattr so a standalone ImageExecutor degrades to the same error
+        # run_container raises, instead of an AttributeError. This check runs
+        # BEFORE the pull decision — same ordering as run_container — so a
+        # GPU task on a non-GPU node fails fast instead of wastefully pulling
+        # a (possibly large) image first.
+        gpu_config = getattr(self, "gpu_config", None)
+        if gpu and not gpu_config:
+            raise MantaDockerError(
+                "GPU support is not enabled for this container manager.",
+                metadata=self.tracer.metadata,
+            )
+
+        if force_pull or await self.image_needs_pull(image):
+            self.tracer.info(f"Pulling image {image}...")
             await self.pull_image_or_raise(image)
 
+        # Copy (not alias) gpu_config's HostConfig — it is shared, long-lived
+        # state on self; mutating it in place would leak NetworkMode/ExtraHosts
+        # onto every future GPU-enabled run.
+        host_config = (
+            dict(gpu_config.get("HostConfig", {})) if (gpu and gpu_config) else {}
+        )
+        host_config["NetworkMode"] = "host"
+        host_config["ExtraHosts"] = ["host.docker.internal:host-gateway"]
+
         env = [f"{k}={v}" for k, v in environment.items()] if environment else None
         config = {
             "Image": image,
-            "HostConfig": {
-                "NetworkMode": "host",
-                "ExtraHosts": ["host.docker.internal:host-gateway"],
-            },
+            "HostConfig": host_config,
             "Env": env,
             "Detach": True,
             "LogConfig": {
                 "type": "json-file",
                 "config": {"max-size": "10m", "max-file": "3"},
             },
+            "Runtime": (gpu_config.get("Runtime") if (gpu and gpu_config) else None),
             "Labels": {
                 "com.docker.compose.project": "manta_tasks",
                 "manta.swarm-id": ID(swarm_id).xid,

manta_node/task_manager.py

@@ -292,6 +292,7 @@ class TaskManager(ContainerManager, ImageExecutor):
                             "SWARM_ID": swarm_id.xid,
                         }
                     ),
+                    gpu=task.gpu,
                 )
                 self.image_task_ids.add(env_tid_pair)
             else:

{manta_node-0.5b0.dev259.dist-info → manta_node-0.5b0.dev264.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: manta_node
-Version: 0.5b0.dev259
+Version: 0.5b0.dev264
 Summary: Node side software of the Manta platform. It is responsible for the communication with the Manta Core and the execution of the tasks.
 Author-email: Hugo Miralles <hugomiralles@manta-tech.io>, Matthew Thompson <matthew.thompson@manta-tech.io>
 License:                     GNU AFFERO GENERAL PUBLIC LICENSE

{manta_node-0.5b0.dev259.dist-info → manta_node-0.5b0.dev264.dist-info}/RECORD

@@ -1,7 +1,7 @@
 manta_node/__init__.py,sha256=bgGWXH3yQVhNf0v1ZGUBAy5Y51fqb0Y-EaLnocbcino,153
 manta_node/__main__.py,sha256=SjrJyuMg_ZFKV6WOVe3MrtWM8Qmu065dkKZYl4n2YLw,1665
 manta_node/node_orchestrator.py,sha256=DW1kiaeybytw656-OJqfhsRiJGMtGS0hfDzIpPyjmdU,17327
-manta_node/task_manager.py,sha256=5C7KN2t985wbs7tf4BP1CcNVKF0A1Av1m8JohyUPoaM,24368
+manta_node/task_manager.py,sha256=aylQs3FAN-zaFd3G99eC_7XWDqBZyVXPJeW-ZqQ5euU,24402
 manta_node/tasks.py,sha256=uBVaWwFq73vHxmoG8Fr4xxkX7rao37KU5hEs79jIrgo,6785
 manta_node/utils.py,sha256=Hw4Iw2jYQYsfVLVJuBMt-_rMFBeml3nCDE2jiiXc8YM,1155
 manta_node/cli/__init__.py,sha256=arzO3mXDJ-E2yeeytbZfN7mCswwxdQ43_QjyeL2E5yE,134
@@ -21,8 +21,8 @@ manta_node/infrastructure/__init__.py,sha256=yM60jKhdaD18BOsLb-0aR9D2XxGvrTnV92o
 manta_node/infrastructure/config/__init__.py,sha256=VnKidPAVfzlZMb5WM-MggtsDLM8tdAFFWlB7WSiawzc,617
 manta_node/infrastructure/config/node_config_manager.py,sha256=u9ga_z7EQ775wkgCVOT0oY4o2uYjAjhCzR1BZmmtNZ0,30605
 manta_node/infrastructure/container/__init__.py,sha256=EO4SndUwmXrHczM4qkleUbxC7lTGz3c3Hd3iGrPg6D8,262
-manta_node/infrastructure/container/docker_adapter.py,sha256=v6tbwSyZp1eGe-rZUat-Y2vSNUKc3kz6zCafsRz7d7M,8395
-manta_node/infrastructure/container/image_executor.py,sha256=ueFl9q4BguZlkBwQMbkdcQ4SQOqMdpLE0Hb4p60GPEs,8499
+manta_node/infrastructure/container/docker_adapter.py,sha256=j4gMPB0FJI_jBRgHX-24er9AgHNU87kb6R4pyKhKaqs,12432
+manta_node/infrastructure/container/image_executor.py,sha256=E3vT3PnfkVxnmM29_r74GKwL_Yr-iYp5PItAGHn-e24,10327
 manta_node/infrastructure/container/manager.py,sha256=ZOMOZaeF0LZUEbcPWGIvpoAfmPmbtzXzayY_v7uP518,18198
 manta_node/infrastructure/filesystem/__init__.py,sha256=nf6Fakp3wuFKjSAm_oihI6WE_yyUY6Uelr7A1GwSvXs,145
 manta_node/infrastructure/filesystem/dataset_manager.py,sha256=8aexrjrkWZ7OCuS0HkjYXZFtg_8Y5WOJ540inK7BYRs,14968
@@ -35,9 +35,9 @@ manta_node/infrastructure/metrics/collector.py,sha256=9ESw__Pc2tV6ldG5yhos2IS7T5
 manta_node/infrastructure/metrics/metrics_collector.py,sha256=WKN3e4q5PANHMrBScaxNSpSoS5ObGLH8QGR0gE1IAOk,10428
 manta_node/infrastructure/mqtt/__init__.py,sha256=riXo1sXgO4YV6DHCmWRqK61BAmuGLdHiVdQFunAJadA,140
 manta_node/infrastructure/mqtt/command_handler.py,sha256=Oj1MORxriUvkBymKsSuXoLqVaOoZWgAcoXy537x35oU,17389
-manta_node-0.5b0.dev259.dist-info/licenses/LICENSE,sha256=P5HojupjKoAusNtHqWOvO-YFolwLFWQ7Uq0XVKK9nFQ,35291
-manta_node-0.5b0.dev259.dist-info/METADATA,sha256=wmdFVZkG6LbgyuLCPVO4OrgfBu3fCDXPYfwMQt2GT_Y,44683
-manta_node-0.5b0.dev259.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-manta_node-0.5b0.dev259.dist-info/entry_points.txt,sha256=C_zcw6mdmztNxVUZGLZ83UH5TwcSuqb2C-rB4e2CQoc,56
-manta_node-0.5b0.dev259.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
-manta_node-0.5b0.dev259.dist-info/RECORD,,
+manta_node-0.5b0.dev264.dist-info/licenses/LICENSE,sha256=P5HojupjKoAusNtHqWOvO-YFolwLFWQ7Uq0XVKK9nFQ,35291
+manta_node-0.5b0.dev264.dist-info/METADATA,sha256=-QVhTQWavTpz2b-lDkW0eE56rGZ5qAZPR6dlCRJa5EA,44683
+manta_node-0.5b0.dev264.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+manta_node-0.5b0.dev264.dist-info/entry_points.txt,sha256=C_zcw6mdmztNxVUZGLZ83UH5TwcSuqb2C-rB4e2CQoc,56
+manta_node-0.5b0.dev264.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
+manta_node-0.5b0.dev264.dist-info/RECORD,,