tuneplane-node 0.3.16__tar.gz → 0.3.19__tar.gz

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/.gitignore

@@ -62,3 +62,6 @@ outputs/
 # Independent public website build
 web/dist-website/
 :memory:.ses
+
+# Shipped, independently packaged plugin artifacts.
+!server/src/tuneplane_server/_plugins/**/*.whl

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuneplane-node
-Version: 0.3.16
+Version: 0.3.19
 Summary: TunePlane node daemon: runs job containers on one machine and reports it to a console.
 License-Expression: AGPL-3.0-only
 License-File: LICENSE

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/pyproject.toml

@@ -10,7 +10,7 @@
 # the local backend runs the same container runtime and allocator on its own host.
 [project]
 name = "tuneplane-node"
-version = "0.3.16"
+version = "0.3.19"
 description = "TunePlane node daemon: runs job containers on one machine and reports it to a console."
 requires-python = ">=3.12"
 # Same licence as the control plane: this is infrastructure an organization runs,

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/src/tuneplane_node/allocator.py

@@ -2,7 +2,7 @@
 
 Who decides "which cards this job gets" when there is no Kubernetes
 ──────────────────────────────────────────────────────────────────────────────
-On the kuberay backend kube-scheduler and the NVIDIA device plugin do it: the Pod
+On the kubernetes backend kube-scheduler and the NVIDIA device plugin do it: the Pod
 declares `nvidia.com/gpu: 4`, the scheduler guarantees no over-subscription, and the
 device plugin injects the actual cards into the container.
 

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/src/tuneplane_node/daemon.py

@@ -47,11 +47,13 @@ import secrets
 import shutil
 import time
 from dataclasses import dataclass, field
+from pathlib import Path
 from weakref import WeakValueDictionary
 
 from fastapi import Depends, FastAPI, HTTPException, Request
 
 from tuneplane_node.allocator import GpuAllocator, NoCapacity
+from tuneplane_node.host_resources import HostResources, claim_host, read_host_capacity
 from tuneplane_node.runtime import LABEL_GPUS, ContainerState, detect_runtime
 from tuneplane_node.runtime import age_seconds as _age_seconds
 from tuneplane_node.wire import PROTOCOL, spec_from_dict, state_to_dict
@@ -244,6 +246,19 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
 
     # ── Launch ──────────────────────────────────────────────────────────────
 
+    def host_capacity():
+        cpus = getattr(settings, "local_allocatable_cpus", 0)
+        memory = getattr(settings, "local_allocatable_memory_gb", 0)
+        if not cpus and not memory:
+            return None
+        if state.runtime.name not in ("docker", "podman"):
+            raise ValueError("CPU admission requires Docker or Podman")
+        return HostResources(cpus, memory * 1024**3)
+
+    def host_lock():
+        # A node-local path: separate machines must never share reservation ledgers.
+        return Path(getattr(settings, "node_state_dir", "~/.tuneplane-node")).expanduser() / "host-allocation.lock"
+
     async def _start_container(spec, run_id: str, requested_gpus: int) -> dict:
         async def _run(gpus: list[int]) -> str:
             # The node writes the pick back itself: GPU passthrough follows the node's
@@ -255,6 +270,25 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
             spec.labels[LABEL_GPUS] = ",".join(str(i) for i in gpus)
             return await state.runtime.run(spec)
 
+        capacity = host_capacity()
+        if requested_gpus == 0 or capacity is not None:
+            request = HostResources.from_labels(spec.labels) or (capacity if requested_gpus > 0 else None)
+            if capacity is None or request is None:
+                raise NoCapacity("CPU-only execution requires configured host capacity and explicit CPU/memory requests")
+            spec.labels.update(request.labels())
+            spec.cpu_limit = str(request.cpus)
+            spec.memory_limit = str(request.memory_bytes)
+            async with claim_host(state.runtime, capacity, request, run_id=run_id, lock_path=host_lock()) as claim:
+                if claim.existing is not None:
+                    return {"phase": "running", "container_id": claim.existing.name, "gpus": claim.existing.gpus, "idempotent": True}
+                if requested_gpus == 0:
+                    cid = await claim.run(lambda: _run([]))
+                    gpus = []
+                else:
+                    gpus, cid = await state.allocator.allocate_and_run(
+                        run_id, requested_gpus, lambda ids: claim.run(lambda: _run(ids))
+                    )
+            return {"phase": "running", "container_id": cid[:12], "gpus": gpus}
         gpus, cid = await state.allocator.allocate_and_run(run_id, requested_gpus, _run)
         return {"phase": "running", "container_id": cid[:12], "gpus": gpus}
 
@@ -294,7 +328,7 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
             # what to do next rather than queueing on the node -- queueing belongs to
             # the console's scheduler, and a second queue must not grow here.
             pending.phase = "failed"
-            pending.error = f"the GPUs were taken during the image pull ({exc}); submit again"
+            pending.error = f"the resources were taken during the image pull ({exc}); submit again"
         except Exception as exc:  # noqa: BLE001
             pending.phase = "failed"
             pending.error = f"image pull or launch failed: {exc}"
@@ -439,6 +473,15 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
         except Exception as exc:  # noqa: BLE001
             return {"ok": False, "runtime": base, "protocol": PROTOCOL,
                     "detail": f"GPU probe failed: {exc}"}
+        host = None
+        try:
+            capacity = host_capacity()
+            if capacity is not None:
+                usage = await read_host_capacity(state.runtime, capacity, lock_path=host_lock())
+                host = {"cpus": capacity.cpus, "memory_bytes": capacity.memory_bytes,
+                        "free_cpus": usage.free_cpus, "free_memory_bytes": usage.free_memory_bytes}
+        except Exception as exc:
+            log.warning("host capacity unavailable: %s", exc)
         # The node's view of the shared storage root. The console reports its
         # own; both should be the same filesystem, and a node that disagrees is
         # exactly the misconfiguration worth surfacing here.
@@ -467,6 +510,7 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
             # console's own cluster_profile says, an H100 node's cards are reported as
             # H200, and every control that works per series stops working.
             "series": _node_series(settings),
+            "host_resources": host,
             "gpus_total": occ.total,
             "gpus_free": max(0, len(occ.free) - reserved),
             "gpus_reserved": reserved,

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/src/tuneplane_node/host_resources.py

@@ -66,12 +66,12 @@ class HostCapacity:
 
 
 def single_host_request(pools) -> HostResources:
-    """The supported local shape, shared by scheduling and container launch."""
+    """The direct single-host shape, shared by scheduling and all CPU executors."""
     if len(pools) != 1 or pools[0].nodes != 1 or not pools[0].cpus or not pools[0].memory_gb:
         raise ValueError("host admission requires one single-machine pool with explicit CPU and memory requests")
     pool = pools[0]
     if pool.scratch_gb:
-        raise ValueError("the local backend does not yet enforce per-job scratch space")
+        raise ValueError("direct single-host jobs do not yet enforce per-job scratch space")
     return HostResources(pool.cpus, pool.memory_gb * 1024**3)
 
 

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/src/tuneplane_node/join.py

@@ -123,7 +123,10 @@ async def join(
         "inventory": inventory,
     }
     async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
-        response = await client.post(f"{base}/api/fleets/{fleet_id}/nodes", json=payload)
+        response = await client.post(
+            f"{base}/api/fleets/{fleet_id}/nodes", json=payload,
+            headers={"User-Agent": _user_agent(node_version)},
+        )
     if response.status_code != 201:
         raise JoinError(_detail(response))
     body = response.json()
@@ -155,7 +158,11 @@ async def beat(
     }
     async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
         response = await client.post(
-            url, json=payload, headers={"Authorization": f"Bearer {identity.credential}"}
+            url, json=payload,
+            headers={
+                "Authorization": f"Bearer {identity.credential}",
+                "User-Agent": _user_agent(node_version),
+            },
         )
     if response.status_code != 200:
         raise JoinError(_detail(response))
@@ -190,6 +197,12 @@ async def heartbeat_forever(
         await asyncio.sleep(interval_s)
 
 
+def _user_agent(node_version: str) -> str:
+    """How the daemon names itself, so a join reads as one in the console's audit trail
+    rather than as `python-httpx`."""
+    return f"tuneplane-node/{node_version}"
+
+
 def _detail(response: httpx.Response) -> str:
     try:
         body = response.json()

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/src/tuneplane_node/runtime.py

@@ -30,6 +30,7 @@ resource boundary between them. Never the default.
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import json
 import logging
 import os
@@ -57,7 +58,38 @@ LABEL_ATTEMPT = "tuneplane.attempt"
 
 #: Container name prefix. A fixed prefix rather than a random name is what makes a
 #: repeated launch of the same run_id produce one job instead of two.
-NAME_PREFIX = "tuneplane-"
+#:
+#: Two letters because the name is read, not parsed: nothing finds a container
+#: by it -- `ps` filters on `LABEL_RUN_ID` -- so all the prefix does is tell an
+#: operator reading `docker ps` on a shared machine which containers are the
+#: platform's. It was `tuneplane-`, ten characters in front of an id that is
+#: itself eighteen, on a name the console and the CLI both print.
+NAME_PREFIX = "tp-"
+
+
+#: Environment names that carry credentials: tokens, keys, vended storage sessions.
+_SECRET_HINTS = ("TOKEN", "SECRET", "KEY", "PASSWORD", "PASSWD", "CREDENTIAL", "STORAGE")
+
+
+def _is_secret(name: str, value: str) -> bool:
+    """Whether an environment value must stay off the command line.
+
+    A presigned URL is a bearer credential whatever its variable is called. A
+    value with a line break cannot be written to an env file and stays in argv.
+    """
+    if "\n" in value or "\r" in value:
+        return False
+    return any(hint in name.upper() for hint in _SECRET_HINTS) or "X-Amz-" in value or "Signature=" in value
+
+
+def _write_env_file(values: dict[str, str]) -> str:
+    """A private (0600) env file for the container CLI to read; the caller removes it."""
+    import tempfile
+
+    fd, path = tempfile.mkstemp(prefix="tuneplane-env-", suffix=".env")
+    with os.fdopen(fd, "w", encoding="utf-8") as handle:
+        handle.write("".join(f"{key}={value}\n" for key, value in values.items()))
+    return path
 
 
 def container_name(run_id: str) -> str:
@@ -242,8 +274,16 @@ class CliRuntime:
             # Override CUDA images that default to exposing every NVIDIA device.
             # Allocation, rather than image or user environment, owns visibility.
             env.update(NVIDIA_VISIBLE_DEVICES="void", CUDA_VISIBLE_DEVICES="")
+        # Secrets never ride in argv: a CLI's command line is readable through
+        # /proc by every local user for as long as the call runs. They go in a
+        # private env file the CLI reads and this process removes right after.
+        secret = {k: v for k, v in env.items() if _is_secret(k, v)}
         for k, v in env.items():
-            args += ["-e", f"{k}={v}"]
+            if k not in secret:
+                args += ["-e", f"{k}={v}"]
+        env_file = _write_env_file(secret) if secret else None
+        if env_file is not None:
+            args += ["--env-file", env_file]
         for k, v in spec.labels.items():
             args += ["--label", f"{k}={v}"]
         for host, cont, ro in spec.mounts:
@@ -272,21 +312,26 @@ class CliRuntime:
         # `vllm` command -- died at once with
         #     vllm: error: unrecognized arguments: -lc python3 -m vllm...
         # which is this `bash -lc` arriving where a subcommand was expected. The
-        # same launch works on kuberay, because a Kubernetes container's
+        # same launch works on kubernetes, because a Kubernetes container's
         # `command` overrides the entrypoint by definition -- so the two backends
         # disagreed about the one field that says what to run, and the process
         # runtime (which execs the argv directly) agreed with Kubernetes.
         #
         # What this gives up is an image's own entrypoint wrapper, such as
         # nvidia_entrypoint.sh on the NGC bases. That wrapper prints a banner and
-        # `exec "$@"`, and the kuberay path has been skipping it since it was
+        # `exec "$@"`, and the kubernetes path has been skipping it since it was
         # written, which is the evidence that nothing depends on it.
         if spec.command:
             args += ["--entrypoint", spec.command[0]]
         args.append(spec.image)
         args += spec.command[1:]
 
-        _, out, _ = await self._exec(*args)
+        try:
+            _, out, _ = await self._exec(*args)
+        finally:
+            if env_file is not None:
+                with contextlib.suppress(OSError):
+                    os.unlink(env_file)
         return out.strip()
 
     async def inspect(self, name: str) -> ContainerState:

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/src/tuneplane_node/settings.py

@@ -17,6 +17,7 @@ import os
 from pathlib import Path
 from typing import Optional
 
+from pydantic import Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from tuneplane.layout import StorageLayout
 
@@ -48,6 +49,15 @@ class NodeSettings(BaseSettings):
     local_gpu_passthrough: str = ""
 
     # ── what cards it has ───────────────────────────────────────────────────
+    local_allocatable_cpus: int = Field(default=0, ge=0)
+    local_allocatable_memory_gb: int = Field(default=0, ge=0)
+
+    @model_validator(mode="after")
+    def _host_capacity_is_paired(self):
+        if bool(self.local_allocatable_cpus) != bool(self.local_allocatable_memory_gb):
+            raise ValueError("local_allocatable_cpus and local_allocatable_memory_gb must both be positive or both zero")
+        return self
+
     local_gpu_count: int = 0
     local_check_gpu_health: bool = True
     local_check_external_gpus: bool = True

{tuneplane_node-0.3.16 → tuneplane_node-0.3.19}/tests/test_runtime.py

@@ -21,6 +21,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import os
 import shutil
 from pathlib import Path
 
@@ -87,10 +88,10 @@ def _argv(runtime: RecordingCli, verb: str) -> list[str]:
 @pytest.mark.parametrize(
     "run_id,expected",
     [
-        ("grpo_demo-alice-20260812", "tuneplane-grpo_demo-alice-20260812"),
-        ("a/b:c", "tuneplane-a-b-c"),
-        ("---", "tuneplane-job"),
-        ("", "tuneplane-job"),
+        ("grpo_demo-alice-20260812", "tp-grpo_demo-alice-20260812"),
+        ("a/b:c", "tp-a-b-c"),
+        ("---", "tp-job"),
+        ("", "tp-job"),
     ],
 )
 def test_a_container_name_is_a_pure_function_of_the_run_id(run_id, expected):
@@ -667,3 +668,27 @@ async def test_strict_inventory_preserves_labels():
 async def test_process_runtime_refuses_host_admission(tmp_path):
     with pytest.raises(RuntimeError_, match="cannot provide enforced host-resource"):
         await ProcessRuntime(tmp_path).ps(strict=True)
+
+
+async def test_credentials_reach_the_container_through_a_private_env_file_never_argv():
+    seen = {}
+
+    class Reading(RecordingCli):
+        async def _exec(self, *args, check=True, timeout=None):
+            if args and args[0] == "run":
+                path = args[list(args).index("--env-file") + 1]
+                seen["mode"] = os.stat(path).st_mode & 0o777
+                with open(path, encoding="utf-8") as handle:
+                    seen["text"] = handle.read()
+                seen["path"] = path
+            return await super()._exec(*args, check=check, timeout=timeout)
+
+    runtime = Reading()
+    await runtime.run(_spec(env={"TUNEPLANE_GOVERNANCE_CREDENTIAL": "tp_gw_secret", "HF_TOKEN": "hf_x",
+                                 "TUNEPLANE_GOVERNANCE_STORAGE": '{"secret_access_key":"s"}',
+                                 "TUNEPLANE_PACKAGE_URL": "https://s3/p?X-Amz-Signature=abc", "NRL_RUN_ID": "run-a"}))
+    argv = " ".join(_argv(runtime, "run"))
+    assert "tp_gw_secret" not in argv and "hf_x" not in argv and "secret_access_key" not in argv and "X-Amz" not in argv
+    assert "NRL_RUN_ID=run-a" in argv
+    assert seen["mode"] == 0o600 and "TUNEPLANE_GOVERNANCE_CREDENTIAL=tp_gw_secret\n" in seen["text"]
+    assert not os.path.exists(seen["path"])