tuneplane-node 0.3.15__tar.gz → 0.3.18__tar.gz

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/.gitignore

@@ -10,6 +10,7 @@ venv/
 .python-version
 .pytest_cache/
 .ruff_cache/
+server/native/target/
 
 # Frontend
 web/node_modules/
@@ -57,3 +58,10 @@ outputs/
 
 # Agent worktrees, created by the harness and never part of a commit.
 .claude/worktrees/
+
+# Independent public website build
+web/dist-website/
+:memory:.ses
+
+# Shipped, independently packaged plugin artifacts.
+!server/src/tuneplane_server/_plugins/**/*.whl

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuneplane-node
-Version: 0.3.15
+Version: 0.3.18
 Summary: TunePlane node daemon: runs job containers on one machine and reports it to a console.
 License-Expression: AGPL-3.0-only
 License-File: LICENSE

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/pyproject.toml

@@ -10,7 +10,7 @@
 # the local backend runs the same container runtime and allocator on its own host.
 [project]
 name = "tuneplane-node"
-version = "0.3.15"
+version = "0.3.18"
 description = "TunePlane node daemon: runs job containers on one machine and reports it to a console."
 requires-python = ">=3.12"
 # Same licence as the control plane: this is infrastructure an organization runs,

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/src/tuneplane_node/daemon.py

@@ -47,11 +47,13 @@ import secrets
 import shutil
 import time
 from dataclasses import dataclass, field
+from pathlib import Path
 from weakref import WeakValueDictionary
 
 from fastapi import Depends, FastAPI, HTTPException, Request
 
 from tuneplane_node.allocator import GpuAllocator, NoCapacity
+from tuneplane_node.host_resources import HostResources, claim_host, read_host_capacity
 from tuneplane_node.runtime import LABEL_GPUS, ContainerState, detect_runtime
 from tuneplane_node.runtime import age_seconds as _age_seconds
 from tuneplane_node.wire import PROTOCOL, spec_from_dict, state_to_dict
@@ -244,6 +246,19 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
 
     # ── Launch ──────────────────────────────────────────────────────────────
 
+    def host_capacity():
+        cpus = getattr(settings, "local_allocatable_cpus", 0)
+        memory = getattr(settings, "local_allocatable_memory_gb", 0)
+        if not cpus and not memory:
+            return None
+        if state.runtime.name not in ("docker", "podman"):
+            raise ValueError("CPU admission requires Docker or Podman")
+        return HostResources(cpus, memory * 1024**3)
+
+    def host_lock():
+        # A node-local path: separate machines must never share reservation ledgers.
+        return Path(getattr(settings, "node_state_dir", "~/.tuneplane-node")).expanduser() / "host-allocation.lock"
+
     async def _start_container(spec, run_id: str, requested_gpus: int) -> dict:
         async def _run(gpus: list[int]) -> str:
             # The node writes the pick back itself: GPU passthrough follows the node's
@@ -255,6 +270,25 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
             spec.labels[LABEL_GPUS] = ",".join(str(i) for i in gpus)
             return await state.runtime.run(spec)
 
+        capacity = host_capacity()
+        if requested_gpus == 0 or capacity is not None:
+            request = HostResources.from_labels(spec.labels) or (capacity if requested_gpus > 0 else None)
+            if capacity is None or request is None:
+                raise NoCapacity("CPU-only execution requires configured host capacity and explicit CPU/memory requests")
+            spec.labels.update(request.labels())
+            spec.cpu_limit = str(request.cpus)
+            spec.memory_limit = str(request.memory_bytes)
+            async with claim_host(state.runtime, capacity, request, run_id=run_id, lock_path=host_lock()) as claim:
+                if claim.existing is not None:
+                    return {"phase": "running", "container_id": claim.existing.name, "gpus": claim.existing.gpus, "idempotent": True}
+                if requested_gpus == 0:
+                    cid = await claim.run(lambda: _run([]))
+                    gpus = []
+                else:
+                    gpus, cid = await state.allocator.allocate_and_run(
+                        run_id, requested_gpus, lambda ids: claim.run(lambda: _run(ids))
+                    )
+            return {"phase": "running", "container_id": cid[:12], "gpus": gpus}
         gpus, cid = await state.allocator.allocate_and_run(run_id, requested_gpus, _run)
         return {"phase": "running", "container_id": cid[:12], "gpus": gpus}
 
@@ -294,7 +328,7 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
             # what to do next rather than queueing on the node -- queueing belongs to
             # the console's scheduler, and a second queue must not grow here.
             pending.phase = "failed"
-            pending.error = f"the GPUs were taken during the image pull ({exc}); submit again"
+            pending.error = f"the resources were taken during the image pull ({exc}); submit again"
         except Exception as exc:  # noqa: BLE001
             pending.phase = "failed"
             pending.error = f"image pull or launch failed: {exc}"
@@ -439,6 +473,15 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
         except Exception as exc:  # noqa: BLE001
             return {"ok": False, "runtime": base, "protocol": PROTOCOL,
                     "detail": f"GPU probe failed: {exc}"}
+        host = None
+        try:
+            capacity = host_capacity()
+            if capacity is not None:
+                usage = await read_host_capacity(state.runtime, capacity, lock_path=host_lock())
+                host = {"cpus": capacity.cpus, "memory_bytes": capacity.memory_bytes,
+                        "free_cpus": usage.free_cpus, "free_memory_bytes": usage.free_memory_bytes}
+        except Exception as exc:
+            log.warning("host capacity unavailable: %s", exc)
         # The node's view of the shared storage root. The console reports its
         # own; both should be the same filesystem, and a node that disagrees is
         # exactly the misconfiguration worth surfacing here.
@@ -467,6 +510,7 @@ def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
             # console's own cluster_profile says, an H100 node's cards are reported as
             # H200, and every control that works per series stops working.
             "series": _node_series(settings),
+            "host_resources": host,
             "gpus_total": occ.total,
             "gpus_free": max(0, len(occ.free) - reserved),
             "gpus_reserved": reserved,

tuneplane_node-0.3.18/src/tuneplane_node/host_resources.py

@@ -0,0 +1,206 @@
+"""Host admission reconstructed from container labels under a shared file lock.
+
+The lock file must be shared by every allocator of the same local runtime and
+must never be unlinked. Container lifetime is reservation lifetime: a restart
+reads the same labels, and an exited container no longer holds CPU or memory.
+Before a container becomes observable, a durable intent covers uncertain startup.
+"""
+from __future__ import annotations
+
+import asyncio
+import fcntl
+import json
+import logging
+import os
+import tempfile
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+from tuneplane_node.allocator import NoCapacity
+from tuneplane_node.runtime import ContainerState
+
+LABEL_CPUS = "tuneplane.host-cpus"
+LABEL_MEMORY = "tuneplane.host-memory-bytes"
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class HostResources:
+    cpus: int
+    memory_bytes: int
+
+    def __post_init__(self):
+        for value in (self.cpus, self.memory_bytes):
+            if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+                raise ValueError("host CPU and memory resources must be positive integers")
+
+    def labels(self) -> dict[str, str]:
+        return {LABEL_CPUS: str(self.cpus), LABEL_MEMORY: str(self.memory_bytes)}
+
+    @classmethod
+    def from_labels(cls, labels: dict[str, str]) -> HostResources | None:
+        try:
+            return cls(int(labels[LABEL_CPUS]), int(labels[LABEL_MEMORY]))
+        except (KeyError, ValueError, TypeError):
+            return None
+
+
+@dataclass(frozen=True)
+class HostCapacity:
+    """One host's observed occupancy, including unresolved launch intents."""
+
+    allocatable: HostResources
+    used_cpus: int
+    used_memory_bytes: int
+    live_runs: frozenset[str]
+    pending_runs: frozenset[str]
+
+    @property
+    def free_cpus(self) -> int:
+        return max(0, self.allocatable.cpus - self.used_cpus)
+
+    @property
+    def free_memory_bytes(self) -> int:
+        return max(0, self.allocatable.memory_bytes - self.used_memory_bytes)
+
+
+def single_host_request(pools) -> HostResources:
+    """The direct single-host shape, shared by scheduling and all CPU executors."""
+    if len(pools) != 1 or pools[0].nodes != 1 or not pools[0].cpus or not pools[0].memory_gb:
+        raise ValueError("host admission requires one single-machine pool with explicit CPU and memory requests")
+    pool = pools[0]
+    if pool.scratch_gb:
+        raise ValueError("direct single-host jobs do not yet enforce per-job scratch space")
+    return HostResources(pool.cpus, pool.memory_gb * 1024**3)
+
+
+def _occupancy(capacity, states, pending) -> HostCapacity:
+    visible = {state.run_id for state in states if state.exists}
+    unresolved = {key: value for key, value in pending.items() if key not in visible}
+    live = [state for state in states if state.exists and state.status != "exited"]
+    held = [HostResources.from_labels(state.labels) or capacity for state in live]
+    held.extend(unresolved.values())
+    return HostCapacity(
+        allocatable=capacity,
+        used_cpus=sum(value.cpus for value in held),
+        used_memory_bytes=sum(value.memory_bytes for value in held),
+        live_runs=frozenset(state.run_id for state in live),
+        pending_runs=frozenset(unresolved),
+    )
+
+
+@asynccontextmanager
+async def _host_lock(lock_path: Path):
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a+b") as lock:
+        while True:
+            try:
+                fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                await asyncio.sleep(0.05)
+        try:
+            yield
+        finally:
+            fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
+
+
+async def read_host_capacity(runtime, capacity: HostResources, *, lock_path: Path) -> HostCapacity:
+    """Read under the launch lock without reconciling or rewriting the ledger."""
+    async with _host_lock(lock_path):
+        states = await runtime.ps(strict=True)
+        return _occupancy(capacity, states, _read_pending(lock_path.with_suffix(".json")))
+
+
+@dataclass
+class HostClaim:
+    existing: ContainerState | None = None
+    started: bool = False
+
+    async def run(self, start):
+        """Mark the point after which a failed call may still create a workload."""
+        if self.existing is not None:
+            raise ValueError("an existing host allocation must not launch another workload")
+        self.started = True
+        return await start()
+
+
+def _read_pending(path: Path) -> dict[str, HostResources]:
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        return {run_id: HostResources(**value) for run_id, value in raw.items()}
+    except FileNotFoundError:
+        return {}
+    except (ValueError, TypeError, AttributeError) as exc:
+        raise NoCapacity("host reservation ledger is unreadable; restore it before admitting jobs") from exc
+
+
+def _write_pending(path: Path, pending: dict[str, HostResources]) -> None:
+    payload = {key: {"cpus": value.cpus, "memory_bytes": value.memory_bytes} for key, value in pending.items()}
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as out:
+        temporary = Path(out.name)
+        try:
+            json.dump(payload, out)
+            out.flush()
+            os.fsync(out.fileno())
+            os.replace(temporary, path)
+            directory = os.open(path.parent, os.O_RDONLY)
+            try:
+                os.fsync(directory)
+            finally:
+                os.close(directory)
+        finally:
+            temporary.unlink(missing_ok=True)
+
+
+@asynccontextmanager
+async def claim_host(runtime, capacity: HostResources, request: HostResources, *, run_id: str, lock_path: Path):
+    """Hold admission through container creation; yield a launch claim or an existing run.
+
+    Unknown live allocations consume the entire host allowance. Unreadable
+    inventory fails closed. The caller must put the request labels on its new
+    container and must not launch another container when an existing run is yielded.
+    """
+    if request.cpus > capacity.cpus or request.memory_bytes > capacity.memory_bytes:
+        raise NoCapacity("the job's host-resource request exceeds this host's allocatable capacity")
+    pending_path = lock_path.with_suffix(".json")
+    async with _host_lock(lock_path):
+        states = await runtime.ps(strict=True)
+        pending = _read_pending(pending_path)
+        # A visible container takes over its durable intent, including when
+        # it already exited. An absent one is an uncertain launch, not free capacity.
+        for state in states:
+            if state.exists:
+                pending.pop(state.run_id, None)
+        _write_pending(pending_path, pending)
+        live = [state for state in states if state.exists and state.status != "exited"]
+        existing = next((state for state in live if state.run_id == run_id), None)
+        if existing is not None:
+            yield HostClaim(existing=existing)
+            return
+        if run_id in pending:
+            raise NoCapacity("this run has an unresolved host reservation; reconcile its launch before retrying")
+        occupancy = _occupancy(capacity, states, pending)
+        if request.cpus > occupancy.free_cpus or request.memory_bytes > occupancy.free_memory_bytes:
+            raise NoCapacity(
+                f"host resources unavailable: need {request.cpus} CPUs and {request.memory_bytes} memory bytes; "
+                f"free {occupancy.free_cpus} CPUs and {occupancy.free_memory_bytes} memory bytes"
+            )
+        pending[run_id] = request
+        _write_pending(pending_path, pending)
+        claim = HostClaim()
+        try:
+            yield claim
+        finally:
+            # Failed/timed-out CLI calls can still create a container later.
+            # Keep their intent across restarts until a workload is observed.
+            try:
+                if not claim.started:
+                    pending.pop(run_id, None)
+                for state in await runtime.ps(strict=True):
+                    if state.exists:
+                        pending.pop(state.run_id, None)
+                _write_pending(pending_path, pending)
+            except Exception:
+                log.exception("host launch outcome is unknown; retaining durable reservations")

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/src/tuneplane_node/runtime.py

@@ -30,6 +30,7 @@ resource boundary between them. Never the default.
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import json
 import logging
 import os
@@ -57,7 +58,38 @@ LABEL_ATTEMPT = "tuneplane.attempt"
 
 #: Container name prefix. A fixed prefix rather than a random name is what makes a
 #: repeated launch of the same run_id produce one job instead of two.
-NAME_PREFIX = "tuneplane-"
+#:
+#: Two letters because the name is read, not parsed: nothing finds a container
+#: by it -- `ps` filters on `LABEL_RUN_ID` -- so all the prefix does is tell an
+#: operator reading `docker ps` on a shared machine which containers are the
+#: platform's. It was `tuneplane-`, ten characters in front of an id that is
+#: itself eighteen, on a name the console and the CLI both print.
+NAME_PREFIX = "tp-"
+
+
+#: Environment names that carry credentials: tokens, keys, vended storage sessions.
+_SECRET_HINTS = ("TOKEN", "SECRET", "KEY", "PASSWORD", "PASSWD", "CREDENTIAL", "STORAGE")
+
+
+def _is_secret(name: str, value: str) -> bool:
+    """Whether an environment value must stay off the command line.
+
+    A presigned URL is a bearer credential whatever its variable is called. A
+    value with a line break cannot be written to an env file and stays in argv.
+    """
+    if "\n" in value or "\r" in value:
+        return False
+    return any(hint in name.upper() for hint in _SECRET_HINTS) or "X-Amz-" in value or "Signature=" in value
+
+
+def _write_env_file(values: dict[str, str]) -> str:
+    """A private (0600) env file for the container CLI to read; the caller removes it."""
+    import tempfile
+
+    fd, path = tempfile.mkstemp(prefix="tuneplane-env-", suffix=".env")
+    with os.fdopen(fd, "w", encoding="utf-8") as handle:
+        handle.write("".join(f"{key}={value}\n" for key, value in values.items()))
+    return path
 
 
 def container_name(run_id: str) -> str:
@@ -162,7 +194,7 @@ class ContainerRuntime(Protocol):
 
     async def remove(self, name: str) -> bool: ...
 
-    async def ps(self) -> list[ContainerState]: ...
+    async def ps(self, *, strict: bool = False) -> list[ContainerState]: ...
 
     async def health(self) -> dict: ...
 
@@ -237,8 +269,21 @@ class CliRuntime:
         #   ttlSecondsAfterFinished.
 
         args += self._gpu_args(spec.gpus)
-        for k, v in spec.env.items():
-            args += ["-e", f"{k}={v}"]
+        env = dict(spec.env)
+        if not spec.gpus:
+            # Override CUDA images that default to exposing every NVIDIA device.
+            # Allocation, rather than image or user environment, owns visibility.
+            env.update(NVIDIA_VISIBLE_DEVICES="void", CUDA_VISIBLE_DEVICES="")
+        # Secrets never ride in argv: a CLI's command line is readable through
+        # /proc by every local user for as long as the call runs. They go in a
+        # private env file the CLI reads and this process removes right after.
+        secret = {k: v for k, v in env.items() if _is_secret(k, v)}
+        for k, v in env.items():
+            if k not in secret:
+                args += ["-e", f"{k}={v}"]
+        env_file = _write_env_file(secret) if secret else None
+        if env_file is not None:
+            args += ["--env-file", env_file]
         for k, v in spec.labels.items():
             args += ["--label", f"{k}={v}"]
         for host, cont, ro in spec.mounts:
@@ -281,7 +326,12 @@ class CliRuntime:
         args.append(spec.image)
         args += spec.command[1:]
 
-        _, out, _ = await self._exec(*args)
+        try:
+            _, out, _ = await self._exec(*args)
+        finally:
+            if env_file is not None:
+                with contextlib.suppress(OSError):
+                    os.unlink(env_file)
         return out.strip()
 
     async def inspect(self, name: str) -> ContainerState:
@@ -317,7 +367,7 @@ class CliRuntime:
         code, _, err = await self._exec("rm", "-f", name, check=False)
         return code == 0 or "no such container" in err.lower()
 
-    async def ps(self) -> list[ContainerState]:
+    async def ps(self, *, strict: bool = False) -> list[ContainerState]:
         """List every container this platform owns, exited ones included.
 
         `--filter label=` filters on the key alone, so what comes back is the platform's
@@ -330,6 +380,18 @@ class CliRuntime:
         names = [n.strip() for n in out.splitlines() if n.strip()]
         if not names:
             return []
+        if strict:
+            async def inspect_required(name: str) -> ContainerState:
+                _, raw, _ = await self._exec("inspect", name)
+                try:
+                    data = json.loads(raw)[0]
+                    if not isinstance(data.get("State"), dict) or not isinstance(data.get("Config"), dict):
+                        raise ValueError("missing container state or configuration")
+                    return _state_from_inspect(name, data)
+                except (ValueError, IndexError, KeyError, TypeError, AttributeError) as exc:
+                    raise RuntimeError_(f"cannot establish resource occupancy for container {name}") from exc
+
+            return list(await asyncio.gather(*(inspect_required(name) for name in names)))
         states = await asyncio.gather(
             *(self.inspect(n) for n in names), return_exceptions=True
         )
@@ -412,6 +474,8 @@ class ProcessRuntime:
 
     async def run(self, spec: ContainerSpec) -> str:
         self.state_dir.mkdir(parents=True, exist_ok=True)
+        if spec.cpu_limit or spec.memory_limit:
+            raise RuntimeError_("the process runtime cannot enforce CPU or memory limits")
         env = {**os.environ, **spec.env}
         if spec.gpus:
             env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in spec.gpus)
@@ -521,7 +585,9 @@ class ProcessRuntime:
             p.unlink(missing_ok=True)
         return True
 
-    async def ps(self) -> list[ContainerState]:
+    async def ps(self, *, strict: bool = False) -> list[ContainerState]:
+        if strict:
+            raise RuntimeError_("the process runtime cannot provide enforced host-resource reservations")
         if not self.state_dir.is_dir():
             return []
         out = []

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/src/tuneplane_node/settings.py

@@ -17,6 +17,7 @@ import os
 from pathlib import Path
 from typing import Optional
 
+from pydantic import Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from tuneplane.layout import StorageLayout
 
@@ -48,6 +49,15 @@ class NodeSettings(BaseSettings):
     local_gpu_passthrough: str = ""
 
     # ── what cards it has ───────────────────────────────────────────────────
+    local_allocatable_cpus: int = Field(default=0, ge=0)
+    local_allocatable_memory_gb: int = Field(default=0, ge=0)
+
+    @model_validator(mode="after")
+    def _host_capacity_is_paired(self):
+        if bool(self.local_allocatable_cpus) != bool(self.local_allocatable_memory_gb):
+            raise ValueError("local_allocatable_cpus and local_allocatable_memory_gb must both be positive or both zero")
+        return self
+
     local_gpu_count: int = 0
     local_check_gpu_health: bool = True
     local_check_external_gpus: bool = True

tuneplane_node-0.3.18/tests/test_host_resources.py

@@ -0,0 +1,221 @@
+"""Host admission survives concurrency, restarts and uncertain container creation."""
+from __future__ import annotations
+
+import asyncio
+import json
+import multiprocessing
+from pathlib import Path
+
+import pytest
+
+from tuneplane_node.allocator import NoCapacity
+from tuneplane_node.host_resources import HostResources, claim_host
+from tuneplane_node.runtime import LABEL_GPUS, LABEL_RUN_ID, ContainerState
+
+CAPACITY = HostResources(4, 8 * 1024**3)
+
+
+class Inventory:
+    def __init__(self, states=()):
+        self.states = list(states)
+
+    async def ps(self, *, strict=False):
+        assert strict
+        return list(self.states)
+
+
+def state(run_id, *, status="running", labels=None):
+    return ContainerState(
+        name=f"tuneplane-{run_id}", exists=True, status=status,
+        labels={LABEL_RUN_ID: run_id, **(CAPACITY.labels() if labels is None else labels)},
+    )
+
+
+def claim(runtime, path, run_id="new", request=CAPACITY):
+    return claim_host(runtime, CAPACITY, request, run_id=run_id, lock_path=path / "host.lock")
+
+
+@pytest.mark.parametrize("status", ["running", "created", "paused", "restarting", "unknown"])
+@pytest.mark.parametrize("labels", [None, {}, {"tuneplane.host-cpus": "bad"}, {LABEL_GPUS: "0"}])
+async def test_live_or_unknown_allocations_hold_capacity_after_restart(tmp_path, status, labels):
+    runtime = Inventory([state("old", status=status, labels=labels)])
+    with pytest.raises(NoCapacity, match="host resources unavailable"):
+        async with claim(runtime, tmp_path):
+            pytest.fail("occupied host admitted another workload")
+
+
+async def test_exit_releases_capacity_and_same_run_is_idempotent(tmp_path):
+    runtime = Inventory([state("old")])
+    async with claim(runtime, tmp_path, "old") as held:
+        assert held.existing.name == "tuneplane-old"
+    runtime.states[0].status = "exited"
+    async with claim(runtime, tmp_path) as held:
+        assert held.existing is None
+
+
+async def test_concurrent_allocators_share_the_file_lock(tmp_path):
+    runtime = Inventory()
+    started, finish = asyncio.Event(), asyncio.Event()
+
+    async def create():
+        started.set()
+        await finish.wait()
+        runtime.states.append(state("first"))
+
+    async def first():
+        async with claim(runtime, tmp_path, "first") as held:
+            await held.run(create)
+
+    async def second():
+        with pytest.raises(NoCapacity):
+            async with claim(runtime, tmp_path, "second"):
+                pytest.fail("concurrent admission oversold the host")
+
+    task = asyncio.create_task(first())
+    await started.wait()
+    waiter = asyncio.create_task(second())
+    await asyncio.sleep(0.02)
+    assert not waiter.done()
+    finish.set()
+    await asyncio.gather(task, waiter)
+
+
+@pytest.mark.parametrize("failure", [TimeoutError, asyncio.CancelledError])
+async def test_uncertain_launch_retains_durable_intent_until_observed(tmp_path, failure):
+    runtime = Inventory()
+
+    async def uncertain():
+        raise failure()
+
+    with pytest.raises(failure):
+        async with claim(runtime, tmp_path, "first") as held:
+            await held.run(uncertain)
+    with pytest.raises(NoCapacity, match="unresolved host reservation"):
+        async with claim(Inventory(), tmp_path, "first"):
+            pytest.fail("same uncertain launch was retried")
+    with pytest.raises(NoCapacity, match="host resources unavailable"):
+        async with claim(Inventory(), tmp_path, "second"):
+            pytest.fail("restart lost a pending reservation")
+    runtime.states.append(state("first", status="exited"))
+    async with claim(runtime, tmp_path, "second"):
+        pass
+    assert json.loads((tmp_path / "host.json").read_text(encoding="utf-8")) == {}
+
+
+async def test_failure_before_runtime_launch_releases_intent(tmp_path):
+    runtime = Inventory()
+    with pytest.raises(NoCapacity, match="GPU"):
+        async with claim(runtime, tmp_path):
+            raise NoCapacity("GPU shortage")
+    async with claim(runtime, tmp_path):
+        pass
+
+
+async def test_corrupt_ledger_blocks_admission(tmp_path):
+    (tmp_path / "host.json").write_text("broken", encoding="utf-8")
+    with pytest.raises(NoCapacity, match="ledger is unreadable"):
+        async with claim(Inventory(), tmp_path):
+            pytest.fail("corrupt ledger admitted a workload")
+
+
+async def test_memory_is_admitted_independently_of_cpu(tmp_path):
+    runtime = Inventory([state("old", labels=HostResources(1, 7 * 1024**3).labels())])
+    with pytest.raises(NoCapacity):
+        async with claim(runtime, tmp_path, request=HostResources(1, 2 * 1024**3)):
+            pytest.fail("memory was oversold despite spare CPUs")
+
+
+def _process_claim(path, ready, outcomes, run_id):
+    """Independent processes must see the same durable uncertain-launch intent."""
+    async def launch():
+        async with claim(Inventory(), Path(path), run_id) as held:
+            async def unobserved():
+                return "runtime accepted the launch"
+            await held.run(unobserved)
+    ready.wait(10)
+    try:
+        asyncio.run(launch())
+        outcomes.put("admitted")
+    except NoCapacity:
+        outcomes.put("blocked")
+
+
+def test_separate_processes_cannot_oversell(tmp_path):
+    context = multiprocessing.get_context("spawn")
+    ready, outcomes = context.Event(), context.Queue()
+    processes = [context.Process(target=_process_claim, args=(str(tmp_path), ready, outcomes, name))
+                 for name in ("first", "second")]
+    try:
+        for process in processes:
+            process.start()
+        ready.set()
+        assert sorted(outcomes.get(timeout=20) for _ in processes) == ["admitted", "blocked"]
+        for process in processes:
+            process.join(timeout=10)
+            assert process.exitcode == 0
+    finally:
+        for process in processes:
+            if process.is_alive():
+                process.terminate()
+            process.join(timeout=5)
+        outcomes.close()
+
+
+async def test_inventory_failure_blocks_without_releasing_a_pending_launch(tmp_path):
+    class Unavailable(Inventory):
+        async def ps(self, *, strict=False):
+            raise RuntimeError("runtime unavailable")
+
+    ledger = tmp_path / "host.json"
+    original = json.dumps({"old": {"cpus": CAPACITY.cpus, "memory_bytes": CAPACITY.memory_bytes}})
+    ledger.write_text(original, encoding="utf-8")
+    with pytest.raises(RuntimeError, match="runtime unavailable"):
+        async with claim(Unavailable(), tmp_path):
+            pytest.fail("unreadable inventory admitted a job")
+    assert ledger.read_text(encoding="utf-8") == original
+
+
+async def test_cancelling_a_lock_waiter_does_not_release_the_active_claim(tmp_path):
+    runtime = Inventory()
+    async with claim(runtime, tmp_path, "owner"):
+        async def wait():
+            async with claim(runtime, tmp_path, "waiter"):
+                pytest.fail("waiter entered a held lock")
+        waiter = asyncio.create_task(wait())
+        await asyncio.sleep(0.02)
+        waiter.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await waiter
+        ledger = json.loads((tmp_path / "host.json").read_text(encoding="utf-8"))
+        assert list(ledger) == ["owner"]
+    async with claim(runtime, tmp_path, "next"):
+        pass
+
+
+async def test_capacity_read_includes_pending_and_does_not_rewrite_ledger(tmp_path):
+    from tuneplane_node.host_resources import read_host_capacity
+
+    path = tmp_path / "host.json"
+    original = json.dumps({
+        "visible": {"cpus": 1, "memory_bytes": 1024**3},
+        "pending": {"cpus": 2, "memory_bytes": 2 * 1024**3},
+    })
+    path.write_text(original, encoding="utf-8")
+    runtime = Inventory([state("visible", labels=HostResources(1, 1024**3).labels())])
+    capacity = await read_host_capacity(runtime, CAPACITY, lock_path=tmp_path / "host.lock")
+    assert capacity.used_cpus == 3 and capacity.used_memory_bytes == 3 * 1024**3
+    assert capacity.free_cpus == 1
+    assert capacity.live_runs == {"visible"} and capacity.pending_runs == {"pending"}
+    assert path.read_text(encoding="utf-8") == original
+    runtime.states[0].status = "exited"
+    capacity = await read_host_capacity(runtime, CAPACITY, lock_path=tmp_path / "host.lock")
+    assert capacity.used_cpus == 2
+    assert path.read_text(encoding="utf-8") == original
+
+
+async def test_empty_capacity_read_does_not_create_reservation_ledger(tmp_path):
+    from tuneplane_node.host_resources import read_host_capacity
+
+    capacity = await read_host_capacity(Inventory(), CAPACITY, lock_path=tmp_path / "host.lock")
+    assert capacity.free_cpus == CAPACITY.cpus
+    assert not (tmp_path / "host.json").exists()

tuneplane_node-0.3.18/tests/test_host_resources_docker.py

@@ -0,0 +1,62 @@
+"""Real Linux Docker acceptance; CI uses a dedicated daemon without GPU devices."""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import uuid
+
+import pytest
+
+from tuneplane_node.allocator import NoCapacity
+from tuneplane_node.host_resources import HostResources, claim_host
+from tuneplane_node.runtime import LABEL_RUN_ID, CliRuntime, ContainerSpec
+
+pytestmark = pytest.mark.skipif(
+    os.environ.get("TUNEPLANE_TEST_HOST_DOCKER") != "1",
+    reason="requires an explicitly enabled dedicated Linux Docker daemon",
+)
+
+
+async def test_real_docker_host_admission_limits_restart_and_release(tmp_path):
+    runtime = CliRuntime("docker", timeout=30)
+    image = "alpine:3.21"
+    await runtime.pull(image, timeout=90)
+    budget = HostResources(1, 128 * 1024**2)
+    names = [f"tuneplane-host-test-{uuid.uuid4().hex}" for _ in range(2)]
+    lock_path = tmp_path / "host.lock"
+
+    async def launch(name, runtime):
+        async with claim_host(runtime, budget, budget, run_id=name, lock_path=lock_path) as held:
+            spec = ContainerSpec(
+                name=name, image=image, command=["sleep", "60"],
+                labels={LABEL_RUN_ID: name, **budget.labels()},
+                cpu_limit="1", memory_limit=str(budget.memory_bytes),
+            )
+            await held.run(lambda: runtime.run(spec))
+        return name
+
+    try:
+        outcomes = await asyncio.gather(*(launch(name, runtime) for name in names), return_exceptions=True)
+        assert sum(isinstance(outcome, NoCapacity) for outcome in outcomes) == 1
+        running = next(outcome for outcome in outcomes if isinstance(outcome, str))
+        waiting = next(name for name in names if name != running)
+        _, raw, _ = await runtime._exec("inspect", running)
+        info = json.loads(raw)[0]
+        assert info["HostConfig"]["NanoCpus"] == 1_000_000_000
+        assert info["HostConfig"]["Memory"] == budget.memory_bytes
+        assert not info["HostConfig"].get("DeviceRequests")
+        assert "NVIDIA_VISIBLE_DEVICES=void" in info["Config"]["Env"]
+        _, raw, _ = await runtime._exec("exec", running, "cat", "/sys/fs/cgroup/memory.max")
+        assert int(raw.strip()) == budget.memory_bytes
+        _, raw, _ = await runtime._exec("exec", running, "cat", "/sys/fs/cgroup/cpu.max")
+        quota, period = map(int, raw.split())
+        assert quota == period
+        restarted = CliRuntime("docker", timeout=30)
+        with pytest.raises(NoCapacity):
+            await launch(waiting, restarted)
+        assert await runtime.stop(running, timeout=1)
+        assert await launch(waiting, restarted) == waiting
+    finally:
+        for name in names:
+            await runtime.remove(name)

{tuneplane_node-0.3.15 → tuneplane_node-0.3.18}/tests/test_runtime.py

@@ -21,6 +21,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import os
 import shutil
 from pathlib import Path
 
@@ -87,10 +88,10 @@ def _argv(runtime: RecordingCli, verb: str) -> list[str]:
 @pytest.mark.parametrize(
     "run_id,expected",
     [
-        ("grpo_demo-alice-20260812", "tuneplane-grpo_demo-alice-20260812"),
-        ("a/b:c", "tuneplane-a-b-c"),
-        ("---", "tuneplane-job"),
-        ("", "tuneplane-job"),
+        ("grpo_demo-alice-20260812", "tp-grpo_demo-alice-20260812"),
+        ("a/b:c", "tp-a-b-c"),
+        ("---", "tp-job"),
+        ("", "tp-job"),
     ],
 )
 def test_a_container_name_is_a_pure_function_of_the_run_id(run_id, expected):
@@ -166,6 +167,39 @@ def test_no_cards_means_no_gpu_flag_at_all():
 # ── run ────────────────────────────────────────────────────────────────────
 
 
+@pytest.mark.parametrize("binary", ["docker", "podman"])
+@pytest.mark.parametrize("env", [{}, {"NVIDIA_VISIBLE_DEVICES": "all", "CUDA_VISIBLE_DEVICES": "0"}])
+async def test_zero_gpu_containers_override_image_and_user_visibility(binary, env):
+    runtime = RecordingCli(binary=binary)
+    spec = _spec(env=env)
+    original = dict(spec.env)
+    await runtime.run(spec)
+    argv = _argv(runtime, "run")
+    assert "--gpus" not in argv and "--device" not in argv
+    assert "NVIDIA_VISIBLE_DEVICES=void" in argv
+    assert "CUDA_VISIBLE_DEVICES=" in argv
+    assert "NVIDIA_VISIBLE_DEVICES=all" not in argv
+    assert "CUDA_VISIBLE_DEVICES=0" not in argv
+    assert spec.env == original
+
+
+async def test_gpu_containers_preserve_their_visibility_environment():
+    runtime = RecordingCli()
+    await runtime.run(_spec(gpus=[2], env={"CUDA_VISIBLE_DEVICES": "0"}))
+    argv = _argv(runtime, "run")
+    assert "--gpus" in argv
+    assert "CUDA_VISIBLE_DEVICES=0" in argv
+    assert "NVIDIA_VISIBLE_DEVICES=void" not in argv
+
+
+@pytest.mark.parametrize("limits", [{"cpu_limit": "4"}, {"memory_limit": "8g"}])
+async def test_process_runtime_refuses_limits_it_cannot_enforce(tmp_path, limits):
+    runtime = ProcessRuntime(tmp_path / "state")
+    with pytest.raises(RuntimeError_, match="cannot enforce CPU or memory"):
+        await runtime.run(_spec(**limits))
+    assert not (tmp_path / "state" / "tuneplane-run-a.json").exists()
+
+
 async def test_the_command_replaces_the_images_entrypoint_rather_than_appending():
     """A Playground session on `vllm/vllm-openai` died at once with
     `vllm: error: unrecognized arguments: -lc python3 -m vllm...` -- our argv
@@ -608,3 +642,53 @@ def test_a_container_that_finished_in_the_future_is_zero_seconds_old():
     """A clock skew between the daemon and the console must not produce a
     negative age, which would compare as younger than every threshold."""
     assert age_seconds("2026-09-12T10:00:00+00:00", 0.0) == 0.0
+
+
+@pytest.mark.parametrize("payload", ["not json", "[]", '[{"State": {}}]', '[{"Config": {}}]'])
+async def test_strict_inventory_refuses_unreadable_occupancy(payload):
+    runtime = RecordingCli(answers={"ps": (0, "tuneplane-a\n", ""), "inspect": (0, payload, "")})
+    with pytest.raises(RuntimeError_, match="cannot establish resource occupancy"):
+        await runtime.ps(strict=True)
+
+
+async def test_strict_inventory_propagates_inspection_failure():
+    runtime = RecordingCli(answers={"ps": (0, "tuneplane-a\n", ""), "inspect": (1, "", "unavailable")})
+    with pytest.raises(RuntimeError_, match="inspect failed"):
+        await runtime.ps(strict=True)
+
+
+async def test_strict_inventory_preserves_labels():
+    runtime = RecordingCli(answers={"ps": (0, "tuneplane-a\n", ""),
+                                   "inspect": (0, _inspect_payload(), "")})
+    states = await runtime.ps(strict=True)
+    assert len(states) == 1 and states[0].exists
+    assert states[0].labels == (await runtime.inspect("tuneplane-a")).labels
+
+
+async def test_process_runtime_refuses_host_admission(tmp_path):
+    with pytest.raises(RuntimeError_, match="cannot provide enforced host-resource"):
+        await ProcessRuntime(tmp_path).ps(strict=True)
+
+
+async def test_credentials_reach_the_container_through_a_private_env_file_never_argv():
+    seen = {}
+
+    class Reading(RecordingCli):
+        async def _exec(self, *args, check=True, timeout=None):
+            if args and args[0] == "run":
+                path = args[list(args).index("--env-file") + 1]
+                seen["mode"] = os.stat(path).st_mode & 0o777
+                with open(path, encoding="utf-8") as handle:
+                    seen["text"] = handle.read()
+                seen["path"] = path
+            return await super()._exec(*args, check=check, timeout=timeout)
+
+    runtime = Reading()
+    await runtime.run(_spec(env={"TUNEPLANE_GOVERNANCE_CREDENTIAL": "tp_gw_secret", "HF_TOKEN": "hf_x",
+                                 "TUNEPLANE_GOVERNANCE_STORAGE": '{"secret_access_key":"s"}',
+                                 "TUNEPLANE_PACKAGE_URL": "https://s3/p?X-Amz-Signature=abc", "NRL_RUN_ID": "run-a"}))
+    argv = " ".join(_argv(runtime, "run"))
+    assert "tp_gw_secret" not in argv and "hf_x" not in argv and "secret_access_key" not in argv and "X-Amz" not in argv
+    assert "NRL_RUN_ID=run-a" in argv
+    assert seen["mode"] == 0o600 and "TUNEPLANE_GOVERNANCE_CREDENTIAL=tp_gw_secret\n" in seen["text"]
+    assert not os.path.exists(seen["path"])