tuneplane-node 0.3.15__tar.gz → 0.3.16__tar.gz

{tuneplane_node-0.3.15 → tuneplane_node-0.3.16}/.gitignore

@@ -10,6 +10,7 @@ venv/
 .python-version
 .pytest_cache/
 .ruff_cache/
+server/native/target/
 
 # Frontend
 web/node_modules/
@@ -57,3 +58,7 @@ outputs/
 
 # Agent worktrees, created by the harness and never part of a commit.
 .claude/worktrees/
+
+# Independent public website build
+web/dist-website/
+:memory:.ses

{tuneplane_node-0.3.15 → tuneplane_node-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuneplane-node
-Version: 0.3.15
+Version: 0.3.16
 Summary: TunePlane node daemon: runs job containers on one machine and reports it to a console.
 License-Expression: AGPL-3.0-only
 License-File: LICENSE

{tuneplane_node-0.3.15 → tuneplane_node-0.3.16}/pyproject.toml

@@ -10,7 +10,7 @@
 # the local backend runs the same container runtime and allocator on its own host.
 [project]
 name = "tuneplane-node"
-version = "0.3.15"
+version = "0.3.16"
 description = "TunePlane node daemon: runs job containers on one machine and reports it to a console."
 requires-python = ">=3.12"
 # Same licence as the control plane: this is infrastructure an organization runs,

tuneplane_node-0.3.16/src/tuneplane_node/host_resources.py

@@ -0,0 +1,206 @@
+"""Host admission reconstructed from container labels under a shared file lock.
+
+The lock file must be shared by every allocator of the same local runtime and
+must never be unlinked. Container lifetime is reservation lifetime: a restart
+reads the same labels, and an exited container no longer holds CPU or memory.
+Before a container becomes observable, a durable intent covers uncertain startup.
+"""
+from __future__ import annotations
+
+import asyncio
+import fcntl
+import json
+import logging
+import os
+import tempfile
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+from tuneplane_node.allocator import NoCapacity
+from tuneplane_node.runtime import ContainerState
+
+LABEL_CPUS = "tuneplane.host-cpus"
+LABEL_MEMORY = "tuneplane.host-memory-bytes"
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class HostResources:
+    cpus: int
+    memory_bytes: int
+
+    def __post_init__(self):
+        for value in (self.cpus, self.memory_bytes):
+            if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+                raise ValueError("host CPU and memory resources must be positive integers")
+
+    def labels(self) -> dict[str, str]:
+        return {LABEL_CPUS: str(self.cpus), LABEL_MEMORY: str(self.memory_bytes)}
+
+    @classmethod
+    def from_labels(cls, labels: dict[str, str]) -> HostResources | None:
+        try:
+            return cls(int(labels[LABEL_CPUS]), int(labels[LABEL_MEMORY]))
+        except (KeyError, ValueError, TypeError):
+            return None
+
+
+@dataclass(frozen=True)
+class HostCapacity:
+    """One host's observed occupancy, including unresolved launch intents."""
+
+    allocatable: HostResources
+    used_cpus: int
+    used_memory_bytes: int
+    live_runs: frozenset[str]
+    pending_runs: frozenset[str]
+
+    @property
+    def free_cpus(self) -> int:
+        return max(0, self.allocatable.cpus - self.used_cpus)
+
+    @property
+    def free_memory_bytes(self) -> int:
+        return max(0, self.allocatable.memory_bytes - self.used_memory_bytes)
+
+
+def single_host_request(pools) -> HostResources:
+    """The supported local shape, shared by scheduling and container launch."""
+    if len(pools) != 1 or pools[0].nodes != 1 or not pools[0].cpus or not pools[0].memory_gb:
+        raise ValueError("host admission requires one single-machine pool with explicit CPU and memory requests")
+    pool = pools[0]
+    if pool.scratch_gb:
+        raise ValueError("the local backend does not yet enforce per-job scratch space")
+    return HostResources(pool.cpus, pool.memory_gb * 1024**3)
+
+
+def _occupancy(capacity, states, pending) -> HostCapacity:
+    visible = {state.run_id for state in states if state.exists}
+    unresolved = {key: value for key, value in pending.items() if key not in visible}
+    live = [state for state in states if state.exists and state.status != "exited"]
+    held = [HostResources.from_labels(state.labels) or capacity for state in live]
+    held.extend(unresolved.values())
+    return HostCapacity(
+        allocatable=capacity,
+        used_cpus=sum(value.cpus for value in held),
+        used_memory_bytes=sum(value.memory_bytes for value in held),
+        live_runs=frozenset(state.run_id for state in live),
+        pending_runs=frozenset(unresolved),
+    )
+
+
+@asynccontextmanager
+async def _host_lock(lock_path: Path):
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a+b") as lock:
+        while True:
+            try:
+                fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                await asyncio.sleep(0.05)
+        try:
+            yield
+        finally:
+            fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
+
+
+async def read_host_capacity(runtime, capacity: HostResources, *, lock_path: Path) -> HostCapacity:
+    """Read under the launch lock without reconciling or rewriting the ledger."""
+    async with _host_lock(lock_path):
+        states = await runtime.ps(strict=True)
+        return _occupancy(capacity, states, _read_pending(lock_path.with_suffix(".json")))
+
+
+@dataclass
+class HostClaim:
+    existing: ContainerState | None = None
+    started: bool = False
+
+    async def run(self, start):
+        """Mark the point after which a failed call may still create a workload."""
+        if self.existing is not None:
+            raise ValueError("an existing host allocation must not launch another workload")
+        self.started = True
+        return await start()
+
+
+def _read_pending(path: Path) -> dict[str, HostResources]:
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        return {run_id: HostResources(**value) for run_id, value in raw.items()}
+    except FileNotFoundError:
+        return {}
+    except (ValueError, TypeError, AttributeError) as exc:
+        raise NoCapacity("host reservation ledger is unreadable; restore it before admitting jobs") from exc
+
+
+def _write_pending(path: Path, pending: dict[str, HostResources]) -> None:
+    payload = {key: {"cpus": value.cpus, "memory_bytes": value.memory_bytes} for key, value in pending.items()}
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as out:
+        temporary = Path(out.name)
+        try:
+            json.dump(payload, out)
+            out.flush()
+            os.fsync(out.fileno())
+            os.replace(temporary, path)
+            directory = os.open(path.parent, os.O_RDONLY)
+            try:
+                os.fsync(directory)
+            finally:
+                os.close(directory)
+        finally:
+            temporary.unlink(missing_ok=True)
+
+
+@asynccontextmanager
+async def claim_host(runtime, capacity: HostResources, request: HostResources, *, run_id: str, lock_path: Path):
+    """Hold admission through container creation; yield a launch claim or an existing run.
+
+    Unknown live allocations consume the entire host allowance. Unreadable
+    inventory fails closed. The caller must put the request labels on its new
+    container and must not launch another container when an existing run is yielded.
+    """
+    if request.cpus > capacity.cpus or request.memory_bytes > capacity.memory_bytes:
+        raise NoCapacity("the job's host-resource request exceeds this host's allocatable capacity")
+    pending_path = lock_path.with_suffix(".json")
+    async with _host_lock(lock_path):
+        states = await runtime.ps(strict=True)
+        pending = _read_pending(pending_path)
+        # A visible container takes over its durable intent, including when
+        # it already exited. An absent one is an uncertain launch, not free capacity.
+        for state in states:
+            if state.exists:
+                pending.pop(state.run_id, None)
+        _write_pending(pending_path, pending)
+        live = [state for state in states if state.exists and state.status != "exited"]
+        existing = next((state for state in live if state.run_id == run_id), None)
+        if existing is not None:
+            yield HostClaim(existing=existing)
+            return
+        if run_id in pending:
+            raise NoCapacity("this run has an unresolved host reservation; reconcile its launch before retrying")
+        occupancy = _occupancy(capacity, states, pending)
+        if request.cpus > occupancy.free_cpus or request.memory_bytes > occupancy.free_memory_bytes:
+            raise NoCapacity(
+                f"host resources unavailable: need {request.cpus} CPUs and {request.memory_bytes} memory bytes; "
+                f"free {occupancy.free_cpus} CPUs and {occupancy.free_memory_bytes} memory bytes"
+            )
+        pending[run_id] = request
+        _write_pending(pending_path, pending)
+        claim = HostClaim()
+        try:
+            yield claim
+        finally:
+            # Failed/timed-out CLI calls can still create a container later.
+            # Keep their intent across restarts until a workload is observed.
+            try:
+                if not claim.started:
+                    pending.pop(run_id, None)
+                for state in await runtime.ps(strict=True):
+                    if state.exists:
+                        pending.pop(state.run_id, None)
+                _write_pending(pending_path, pending)
+            except Exception:
+                log.exception("host launch outcome is unknown; retaining durable reservations")

{tuneplane_node-0.3.15 → tuneplane_node-0.3.16}/src/tuneplane_node/runtime.py

@@ -162,7 +162,7 @@ class ContainerRuntime(Protocol):
 
     async def remove(self, name: str) -> bool: ...
 
-    async def ps(self) -> list[ContainerState]: ...
+    async def ps(self, *, strict: bool = False) -> list[ContainerState]: ...
 
     async def health(self) -> dict: ...
 
@@ -237,7 +237,12 @@ class CliRuntime:
         #   ttlSecondsAfterFinished.
 
         args += self._gpu_args(spec.gpus)
-        for k, v in spec.env.items():
+        env = dict(spec.env)
+        if not spec.gpus:
+            # Override CUDA images that default to exposing every NVIDIA device.
+            # Allocation, rather than image or user environment, owns visibility.
+            env.update(NVIDIA_VISIBLE_DEVICES="void", CUDA_VISIBLE_DEVICES="")
+        for k, v in env.items():
             args += ["-e", f"{k}={v}"]
         for k, v in spec.labels.items():
             args += ["--label", f"{k}={v}"]
@@ -317,7 +322,7 @@ class CliRuntime:
         code, _, err = await self._exec("rm", "-f", name, check=False)
         return code == 0 or "no such container" in err.lower()
 
-    async def ps(self) -> list[ContainerState]:
+    async def ps(self, *, strict: bool = False) -> list[ContainerState]:
         """List every container this platform owns, exited ones included.
 
         `--filter label=` filters on the key alone, so what comes back is the platform's
@@ -330,6 +335,18 @@ class CliRuntime:
         names = [n.strip() for n in out.splitlines() if n.strip()]
         if not names:
             return []
+        if strict:
+            async def inspect_required(name: str) -> ContainerState:
+                _, raw, _ = await self._exec("inspect", name)
+                try:
+                    data = json.loads(raw)[0]
+                    if not isinstance(data.get("State"), dict) or not isinstance(data.get("Config"), dict):
+                        raise ValueError("missing container state or configuration")
+                    return _state_from_inspect(name, data)
+                except (ValueError, IndexError, KeyError, TypeError, AttributeError) as exc:
+                    raise RuntimeError_(f"cannot establish resource occupancy for container {name}") from exc
+
+            return list(await asyncio.gather(*(inspect_required(name) for name in names)))
         states = await asyncio.gather(
             *(self.inspect(n) for n in names), return_exceptions=True
         )
@@ -412,6 +429,8 @@ class ProcessRuntime:
 
     async def run(self, spec: ContainerSpec) -> str:
         self.state_dir.mkdir(parents=True, exist_ok=True)
+        if spec.cpu_limit or spec.memory_limit:
+            raise RuntimeError_("the process runtime cannot enforce CPU or memory limits")
         env = {**os.environ, **spec.env}
         if spec.gpus:
             env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in spec.gpus)
@@ -521,7 +540,9 @@ class ProcessRuntime:
             p.unlink(missing_ok=True)
         return True
 
-    async def ps(self) -> list[ContainerState]:
+    async def ps(self, *, strict: bool = False) -> list[ContainerState]:
+        if strict:
+            raise RuntimeError_("the process runtime cannot provide enforced host-resource reservations")
         if not self.state_dir.is_dir():
             return []
         out = []

tuneplane_node-0.3.16/tests/test_host_resources.py

@@ -0,0 +1,221 @@
+"""Host admission survives concurrency, restarts and uncertain container creation."""
+from __future__ import annotations
+
+import asyncio
+import json
+import multiprocessing
+from pathlib import Path
+
+import pytest
+
+from tuneplane_node.allocator import NoCapacity
+from tuneplane_node.host_resources import HostResources, claim_host
+from tuneplane_node.runtime import LABEL_GPUS, LABEL_RUN_ID, ContainerState
+
+CAPACITY = HostResources(4, 8 * 1024**3)
+
+
+class Inventory:
+    def __init__(self, states=()):
+        self.states = list(states)
+
+    async def ps(self, *, strict=False):
+        assert strict
+        return list(self.states)
+
+
+def state(run_id, *, status="running", labels=None):
+    return ContainerState(
+        name=f"tuneplane-{run_id}", exists=True, status=status,
+        labels={LABEL_RUN_ID: run_id, **(CAPACITY.labels() if labels is None else labels)},
+    )
+
+
+def claim(runtime, path, run_id="new", request=CAPACITY):
+    return claim_host(runtime, CAPACITY, request, run_id=run_id, lock_path=path / "host.lock")
+
+
+@pytest.mark.parametrize("status", ["running", "created", "paused", "restarting", "unknown"])
+@pytest.mark.parametrize("labels", [None, {}, {"tuneplane.host-cpus": "bad"}, {LABEL_GPUS: "0"}])
+async def test_live_or_unknown_allocations_hold_capacity_after_restart(tmp_path, status, labels):
+    runtime = Inventory([state("old", status=status, labels=labels)])
+    with pytest.raises(NoCapacity, match="host resources unavailable"):
+        async with claim(runtime, tmp_path):
+            pytest.fail("occupied host admitted another workload")
+
+
+async def test_exit_releases_capacity_and_same_run_is_idempotent(tmp_path):
+    runtime = Inventory([state("old")])
+    async with claim(runtime, tmp_path, "old") as held:
+        assert held.existing.name == "tuneplane-old"
+    runtime.states[0].status = "exited"
+    async with claim(runtime, tmp_path) as held:
+        assert held.existing is None
+
+
+async def test_concurrent_allocators_share_the_file_lock(tmp_path):
+    runtime = Inventory()
+    started, finish = asyncio.Event(), asyncio.Event()
+
+    async def create():
+        started.set()
+        await finish.wait()
+        runtime.states.append(state("first"))
+
+    async def first():
+        async with claim(runtime, tmp_path, "first") as held:
+            await held.run(create)
+
+    async def second():
+        with pytest.raises(NoCapacity):
+            async with claim(runtime, tmp_path, "second"):
+                pytest.fail("concurrent admission oversold the host")
+
+    task = asyncio.create_task(first())
+    await started.wait()
+    waiter = asyncio.create_task(second())
+    await asyncio.sleep(0.02)
+    assert not waiter.done()
+    finish.set()
+    await asyncio.gather(task, waiter)
+
+
+@pytest.mark.parametrize("failure", [TimeoutError, asyncio.CancelledError])
+async def test_uncertain_launch_retains_durable_intent_until_observed(tmp_path, failure):
+    runtime = Inventory()
+
+    async def uncertain():
+        raise failure()
+
+    with pytest.raises(failure):
+        async with claim(runtime, tmp_path, "first") as held:
+            await held.run(uncertain)
+    with pytest.raises(NoCapacity, match="unresolved host reservation"):
+        async with claim(Inventory(), tmp_path, "first"):
+            pytest.fail("same uncertain launch was retried")
+    with pytest.raises(NoCapacity, match="host resources unavailable"):
+        async with claim(Inventory(), tmp_path, "second"):
+            pytest.fail("restart lost a pending reservation")
+    runtime.states.append(state("first", status="exited"))
+    async with claim(runtime, tmp_path, "second"):
+        pass
+    assert json.loads((tmp_path / "host.json").read_text(encoding="utf-8")) == {}
+
+
+async def test_failure_before_runtime_launch_releases_intent(tmp_path):
+    runtime = Inventory()
+    with pytest.raises(NoCapacity, match="GPU"):
+        async with claim(runtime, tmp_path):
+            raise NoCapacity("GPU shortage")
+    async with claim(runtime, tmp_path):
+        pass
+
+
+async def test_corrupt_ledger_blocks_admission(tmp_path):
+    (tmp_path / "host.json").write_text("broken", encoding="utf-8")
+    with pytest.raises(NoCapacity, match="ledger is unreadable"):
+        async with claim(Inventory(), tmp_path):
+            pytest.fail("corrupt ledger admitted a workload")
+
+
+async def test_memory_is_admitted_independently_of_cpu(tmp_path):
+    runtime = Inventory([state("old", labels=HostResources(1, 7 * 1024**3).labels())])
+    with pytest.raises(NoCapacity):
+        async with claim(runtime, tmp_path, request=HostResources(1, 2 * 1024**3)):
+            pytest.fail("memory was oversold despite spare CPUs")
+
+
+def _process_claim(path, ready, outcomes, run_id):
+    """Independent processes must see the same durable uncertain-launch intent."""
+    async def launch():
+        async with claim(Inventory(), Path(path), run_id) as held:
+            async def unobserved():
+                return "runtime accepted the launch"
+            await held.run(unobserved)
+    ready.wait(10)
+    try:
+        asyncio.run(launch())
+        outcomes.put("admitted")
+    except NoCapacity:
+        outcomes.put("blocked")
+
+
+def test_separate_processes_cannot_oversell(tmp_path):
+    context = multiprocessing.get_context("spawn")
+    ready, outcomes = context.Event(), context.Queue()
+    processes = [context.Process(target=_process_claim, args=(str(tmp_path), ready, outcomes, name))
+                 for name in ("first", "second")]
+    try:
+        for process in processes:
+            process.start()
+        ready.set()
+        assert sorted(outcomes.get(timeout=20) for _ in processes) == ["admitted", "blocked"]
+        for process in processes:
+            process.join(timeout=10)
+            assert process.exitcode == 0
+    finally:
+        for process in processes:
+            if process.is_alive():
+                process.terminate()
+            process.join(timeout=5)
+        outcomes.close()
+
+
+async def test_inventory_failure_blocks_without_releasing_a_pending_launch(tmp_path):
+    class Unavailable(Inventory):
+        async def ps(self, *, strict=False):
+            raise RuntimeError("runtime unavailable")
+
+    ledger = tmp_path / "host.json"
+    original = json.dumps({"old": {"cpus": CAPACITY.cpus, "memory_bytes": CAPACITY.memory_bytes}})
+    ledger.write_text(original, encoding="utf-8")
+    with pytest.raises(RuntimeError, match="runtime unavailable"):
+        async with claim(Unavailable(), tmp_path):
+            pytest.fail("unreadable inventory admitted a job")
+    assert ledger.read_text(encoding="utf-8") == original
+
+
+async def test_cancelling_a_lock_waiter_does_not_release_the_active_claim(tmp_path):
+    runtime = Inventory()
+    async with claim(runtime, tmp_path, "owner"):
+        async def wait():
+            async with claim(runtime, tmp_path, "waiter"):
+                pytest.fail("waiter entered a held lock")
+        waiter = asyncio.create_task(wait())
+        await asyncio.sleep(0.02)
+        waiter.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await waiter
+        ledger = json.loads((tmp_path / "host.json").read_text(encoding="utf-8"))
+        assert list(ledger) == ["owner"]
+    async with claim(runtime, tmp_path, "next"):
+        pass
+
+
+async def test_capacity_read_includes_pending_and_does_not_rewrite_ledger(tmp_path):
+    from tuneplane_node.host_resources import read_host_capacity
+
+    path = tmp_path / "host.json"
+    original = json.dumps({
+        "visible": {"cpus": 1, "memory_bytes": 1024**3},
+        "pending": {"cpus": 2, "memory_bytes": 2 * 1024**3},
+    })
+    path.write_text(original, encoding="utf-8")
+    runtime = Inventory([state("visible", labels=HostResources(1, 1024**3).labels())])
+    capacity = await read_host_capacity(runtime, CAPACITY, lock_path=tmp_path / "host.lock")
+    assert capacity.used_cpus == 3 and capacity.used_memory_bytes == 3 * 1024**3
+    assert capacity.free_cpus == 1
+    assert capacity.live_runs == {"visible"} and capacity.pending_runs == {"pending"}
+    assert path.read_text(encoding="utf-8") == original
+    runtime.states[0].status = "exited"
+    capacity = await read_host_capacity(runtime, CAPACITY, lock_path=tmp_path / "host.lock")
+    assert capacity.used_cpus == 2
+    assert path.read_text(encoding="utf-8") == original
+
+
+async def test_empty_capacity_read_does_not_create_reservation_ledger(tmp_path):
+    from tuneplane_node.host_resources import read_host_capacity
+
+    capacity = await read_host_capacity(Inventory(), CAPACITY, lock_path=tmp_path / "host.lock")
+    assert capacity.free_cpus == CAPACITY.cpus
+    assert not (tmp_path / "host.json").exists()

tuneplane_node-0.3.16/tests/test_host_resources_docker.py

@@ -0,0 +1,62 @@
+"""Real Linux Docker acceptance; CI uses a dedicated daemon without GPU devices."""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import uuid
+
+import pytest
+
+from tuneplane_node.allocator import NoCapacity
+from tuneplane_node.host_resources import HostResources, claim_host
+from tuneplane_node.runtime import LABEL_RUN_ID, CliRuntime, ContainerSpec
+
+pytestmark = pytest.mark.skipif(
+    os.environ.get("TUNEPLANE_TEST_HOST_DOCKER") != "1",
+    reason="requires an explicitly enabled dedicated Linux Docker daemon",
+)
+
+
+async def test_real_docker_host_admission_limits_restart_and_release(tmp_path):
+    runtime = CliRuntime("docker", timeout=30)
+    image = "alpine:3.21"
+    await runtime.pull(image, timeout=90)
+    budget = HostResources(1, 128 * 1024**2)
+    names = [f"tuneplane-host-test-{uuid.uuid4().hex}" for _ in range(2)]
+    lock_path = tmp_path / "host.lock"
+
+    async def launch(name, runtime):
+        async with claim_host(runtime, budget, budget, run_id=name, lock_path=lock_path) as held:
+            spec = ContainerSpec(
+                name=name, image=image, command=["sleep", "60"],
+                labels={LABEL_RUN_ID: name, **budget.labels()},
+                cpu_limit="1", memory_limit=str(budget.memory_bytes),
+            )
+            await held.run(lambda: runtime.run(spec))
+        return name
+
+    try:
+        outcomes = await asyncio.gather(*(launch(name, runtime) for name in names), return_exceptions=True)
+        assert sum(isinstance(outcome, NoCapacity) for outcome in outcomes) == 1
+        running = next(outcome for outcome in outcomes if isinstance(outcome, str))
+        waiting = next(name for name in names if name != running)
+        _, raw, _ = await runtime._exec("inspect", running)
+        info = json.loads(raw)[0]
+        assert info["HostConfig"]["NanoCpus"] == 1_000_000_000
+        assert info["HostConfig"]["Memory"] == budget.memory_bytes
+        assert not info["HostConfig"].get("DeviceRequests")
+        assert "NVIDIA_VISIBLE_DEVICES=void" in info["Config"]["Env"]
+        _, raw, _ = await runtime._exec("exec", running, "cat", "/sys/fs/cgroup/memory.max")
+        assert int(raw.strip()) == budget.memory_bytes
+        _, raw, _ = await runtime._exec("exec", running, "cat", "/sys/fs/cgroup/cpu.max")
+        quota, period = map(int, raw.split())
+        assert quota == period
+        restarted = CliRuntime("docker", timeout=30)
+        with pytest.raises(NoCapacity):
+            await launch(waiting, restarted)
+        assert await runtime.stop(running, timeout=1)
+        assert await launch(waiting, restarted) == waiting
+    finally:
+        for name in names:
+            await runtime.remove(name)

{tuneplane_node-0.3.15 → tuneplane_node-0.3.16}/tests/test_runtime.py

@@ -166,6 +166,39 @@ def test_no_cards_means_no_gpu_flag_at_all():
 # ── run ────────────────────────────────────────────────────────────────────
 
 
+@pytest.mark.parametrize("binary", ["docker", "podman"])
+@pytest.mark.parametrize("env", [{}, {"NVIDIA_VISIBLE_DEVICES": "all", "CUDA_VISIBLE_DEVICES": "0"}])
+async def test_zero_gpu_containers_override_image_and_user_visibility(binary, env):
+    runtime = RecordingCli(binary=binary)
+    spec = _spec(env=env)
+    original = dict(spec.env)
+    await runtime.run(spec)
+    argv = _argv(runtime, "run")
+    assert "--gpus" not in argv and "--device" not in argv
+    assert "NVIDIA_VISIBLE_DEVICES=void" in argv
+    assert "CUDA_VISIBLE_DEVICES=" in argv
+    assert "NVIDIA_VISIBLE_DEVICES=all" not in argv
+    assert "CUDA_VISIBLE_DEVICES=0" not in argv
+    assert spec.env == original
+
+
+async def test_gpu_containers_preserve_their_visibility_environment():
+    runtime = RecordingCli()
+    await runtime.run(_spec(gpus=[2], env={"CUDA_VISIBLE_DEVICES": "0"}))
+    argv = _argv(runtime, "run")
+    assert "--gpus" in argv
+    assert "CUDA_VISIBLE_DEVICES=0" in argv
+    assert "NVIDIA_VISIBLE_DEVICES=void" not in argv
+
+
+@pytest.mark.parametrize("limits", [{"cpu_limit": "4"}, {"memory_limit": "8g"}])
+async def test_process_runtime_refuses_limits_it_cannot_enforce(tmp_path, limits):
+    runtime = ProcessRuntime(tmp_path / "state")
+    with pytest.raises(RuntimeError_, match="cannot enforce CPU or memory"):
+        await runtime.run(_spec(**limits))
+    assert not (tmp_path / "state" / "tuneplane-run-a.json").exists()
+
+
 async def test_the_command_replaces_the_images_entrypoint_rather_than_appending():
     """A Playground session on `vllm/vllm-openai` died at once with
     `vllm: error: unrecognized arguments: -lc python3 -m vllm...` -- our argv
@@ -608,3 +641,29 @@ def test_a_container_that_finished_in_the_future_is_zero_seconds_old():
     """A clock skew between the daemon and the console must not produce a
     negative age, which would compare as younger than every threshold."""
     assert age_seconds("2026-09-12T10:00:00+00:00", 0.0) == 0.0
+
+
+@pytest.mark.parametrize("payload", ["not json", "[]", '[{"State": {}}]', '[{"Config": {}}]'])
+async def test_strict_inventory_refuses_unreadable_occupancy(payload):
+    runtime = RecordingCli(answers={"ps": (0, "tuneplane-a\n", ""), "inspect": (0, payload, "")})
+    with pytest.raises(RuntimeError_, match="cannot establish resource occupancy"):
+        await runtime.ps(strict=True)
+
+
+async def test_strict_inventory_propagates_inspection_failure():
+    runtime = RecordingCli(answers={"ps": (0, "tuneplane-a\n", ""), "inspect": (1, "", "unavailable")})
+    with pytest.raises(RuntimeError_, match="inspect failed"):
+        await runtime.ps(strict=True)
+
+
+async def test_strict_inventory_preserves_labels():
+    runtime = RecordingCli(answers={"ps": (0, "tuneplane-a\n", ""),
+                                   "inspect": (0, _inspect_payload(), "")})
+    states = await runtime.ps(strict=True)
+    assert len(states) == 1 and states[0].exists
+    assert states[0].labels == (await runtime.inspect("tuneplane-a")).labels
+
+
+async def test_process_runtime_refuses_host_admission(tmp_path):
+    with pytest.raises(RuntimeError_, match="cannot provide enforced host-resource"):
+        await ProcessRuntime(tmp_path).ps(strict=True)