tuneplane-node 0.3.15__py3-none-any.whl

tuneplane_node/__init__.py

@@ -0,0 +1 @@
+"""TunePlane node agent: what a machine needs to be a Fleet node."""

tuneplane_node/allocator.py

@@ -0,0 +1,596 @@
+"""Physical GPU allocator.
+
+Who decides "which cards this job gets" when there is no Kubernetes
+──────────────────────────────────────────────────────────────────────────────
+On the kuberay backend kube-scheduler and the NVIDIA device plugin do it: the Pod
+declares `nvidia.com/gpu: 4`, the scheduler guarantees no over-subscription, and the
+device plugin injects the actual cards into the container.
+
+The local backend has neither. What their absence costs: **two jobs placed on the
+same card**, both going OOM or slowing each other down, and the symptom is random --
+it depends on which one asks for memory first. That is the most typical failure
+without a device plugin, and the hardest to diagnose, so this layer cannot be
+skipped.
+
+Why the source of truth is container labels rather than a table in memory
+──────────────────────────────────────────────────────────────────────────────
+The console restarts (upgrades, crashes, migrations) while a training job runs for
+hours or days. An in-memory table is empty after a restart, so a new job is placed
+on a card already in use -- exactly the failure above.
+
+Writing "who holds which card" into a container label (`tuneplane.gpus`) means one
+`docker ps` rebuilds the whole picture after a restart. This is the same judgement
+as "object storage is the truth, no DB table" on the datasets side: state lives
+**where it already belongs**, so there is no second copy to keep in sync.
+
+Use from outside the platform can only be seen through nvidia-smi, and that has a
+fatal blind spot
+──────────────────────────────────────────────────────────────────────────────
+Container labels cover only the jobs the platform started. A python somebody ran by
+hand over ssh, or a container another tool started, the platform knows nothing
+about -- only nvidia-smi can be asked.
+
+But the NVIDIA driver **does not honour PID namespaces**: `--query-compute-apps`
+reports host PIDs, which do not exist inside the container the console runs in, so
+the process list comes back empty. Which is to say that under a production
+deployment (docker-compose) the "look at the processes" route is effectively off,
+and a full card is handed out as idle. So occupancy detection has to look at
+**memory used** as well (a per-device counter, the same inside a container as
+outside), and when detection is unavailable altogether it has to say so
+(`external_probe_ok`) instead of silently reporting the cards free.
+"""
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import shutil
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+def _raw_series(profile: str, gpu_name: str) -> str:
+    """What a node reports when nobody has told it about hardware series.
+
+    The registry belongs to the console, so a node hands back the raw strings
+    and the console maps them. Lower-cased only, because a series id is
+    lower-case everywhere else and a node that shouted would look like a
+    different series to a case-sensitive comparison.
+    """
+    return (profile or gpu_name or "").strip().lower()
+
+
+def _lock_token() -> str:
+    """A value unique to this holder of the allocation lock.
+
+    Time-prefixed so a stuck lock says roughly when it was taken. Inlined rather
+    than imported from the console's redis helpers: three lines is not worth a
+    dependency from the node package back into the control plane.
+    """
+    import secrets
+    import time as _time
+
+    return f"{int(_time.time() * 1000)}-{secrets.token_hex(8)}"
+
+log = logging.getLogger(__name__)
+
+
+class NoCapacity(RuntimeError):
+    """Not enough free cards.
+
+    A type of its own because callers have to treat it differently: this is **not an
+    error**, it is "not your turn yet". The dequeue loop catches it and leaves the job
+    in the queue to try again next round rather than marking it failed.
+    """
+
+
+@dataclass
+class GpuOccupancy:
+    """What is physically held at one moment."""
+
+    total: int
+    #: card index -> run_id (the platform's own jobs)
+    by_job: dict[int, str] = field(default_factory=dict)
+    #: indices of cards held from outside the platform (somebody's hand-run python)
+    external: set[int] = field(default_factory=set)
+    #: card index -> why it is unhealthy (pending retired pages / uncorrected errors /
+    #: hardware slowdown). Such a card exists but must not be handed out: a job placed
+    #: on it crashes mid-training or silently runs slow.
+    unhealthy: dict[int, str] = field(default_factory=dict)
+    #: Whether detection of use from outside the platform worked. When False `external`
+    #: is always empty, which **does not mean nobody holds a card** -- it has to be
+    #: visible, or the chain "free count inflated -> a full card handed out -> the job
+    #: goes OOM" leaves no trace at all.
+    external_probe_ok: bool = True
+
+    @property
+    def busy(self) -> set[int]:
+        return set(self.by_job) | self.external | set(self.unhealthy)
+
+    @property
+    def free(self) -> list[int]:
+        return [i for i in range(self.total) if i not in self.busy]
+
+    def explain(self) -> str:
+        """Plain-language explanation for the admission gate and the /scheduling API."""
+        parts = [f"{self.total} cards total, {len(self.free)} free"]
+        if self.by_job:
+            parts.append(f"{len(self.by_job)} held by platform jobs")
+        if self.external:
+            ext = ",".join(str(i) for i in sorted(self.external))
+            parts.append(f"{len(self.external)} held by processes outside the platform (cards {ext})")
+        if self.unhealthy:
+            bad = "; ".join(f"card {i}: {r}" for i, r in sorted(self.unhealthy.items()))
+            parts.append(f"{len(self.unhealthy)} unhealthy ({bad})")
+        if not self.external_probe_ok:
+            parts.append("cannot probe use from outside the platform (nvidia-smi not found), "
+                         "so the free count may be too high")
+        return "; ".join(parts)
+
+
+async def _smi(*args: str) -> Optional[str]:
+    """Run nvidia-smi once and return its stdout; None when it cannot be run.
+
+    ★ None and an empty string are two different things: None is "detection is broken,
+    I do not know whether anyone holds a card", empty is "asked, and nobody does". The
+    occupancy view must keep them apart -- reading "unknown" as "nobody" hands out a
+    full card as idle, which is the failure this module exists to prevent.
+    """
+    if not shutil.which("nvidia-smi"):
+        return None
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "nvidia-smi", *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.DEVNULL,
+        )
+        out, _ = await asyncio.wait_for(proc.communicate(), timeout=10)
+    except (OSError, asyncio.TimeoutError) as e:
+        log.debug("nvidia-smi %s failed: %s", " ".join(args), e)
+        return None
+    if proc.returncode != 0:
+        log.debug("nvidia-smi %s exited %s", " ".join(args), proc.returncode)
+        return None
+    return out.decode(errors="replace")
+
+
+async def _nvidia_smi_busy() -> set[int]:
+    """Which cards hold a process that **can be seen**.
+
+    ⚠ This probe has a blind spot inside a container and cannot decide on its own --
+    see _nvidia_smi_memory_busy. Its value is that it attributes use to a specific
+    process, which is more precise than memory used when the console runs on the host.
+    """
+    out = await _smi("--query-compute-apps=gpu_uuid", "--format=csv,noheader")
+    if not out:
+        return set()
+    uuids = {line.strip() for line in out.splitlines() if line.strip()}
+    if not uuids:
+        return set()
+    return await _uuids_to_indexes(uuids)
+
+
+async def _uuids_to_indexes(uuids: set[str]) -> set[int]:
+    """GPU UUID -> card index.
+
+    `--query-compute-apps` gives UUIDs and no index, so the mapping takes a second
+    query. The order processes appear in cannot stand in for the index: that is wrong
+    as soon as there are several processes, or MIG.
+    """
+    out = await _smi("--query-gpu=index,uuid", "--format=csv,noheader")
+    if not out:
+        return set()
+    busy: set[int] = set()
+    for line in out.splitlines():
+        parts = [p.strip() for p in line.split(",")]
+        if len(parts) >= 2 and parts[0].isdigit() and parts[1] in uuids:
+            busy.add(int(parts[0]))
+    return busy
+
+
+#: Memory used above this (MiB) counts the card as in use. An idle card usually sits
+#: at single-digit MiB while a training process holds tens of GB -- the threshold sits
+#: between them, with room for the driver and ECC baseline.
+_MEM_BUSY_MIB = 512
+
+
+def parse_gpu_memory_busy(text: str, threshold_mib: int = _MEM_BUSY_MIB) -> set[int]:
+    """`--query-gpu=index,memory.used --format=...,nounits` output -> indices in use."""
+    busy: set[int] = set()
+    for line in text.splitlines():
+        parts = [p.strip() for p in line.split(",")]
+        if len(parts) < 2 or not parts[0].isdigit():
+            continue
+        used = parts[1].split()[0] if parts[1] else ""
+        if used.isdigit() and int(used) >= threshold_mib:
+            busy.add(int(parts[0]))
+    return busy
+
+
+async def _nvidia_smi_memory_busy() -> Optional[set[int]]:
+    """Which cards have memory in use. None = detection unavailable.
+
+    Why the process list (_nvidia_smi_busy) is not enough on its own: the NVIDIA driver
+    **does not honour PID namespaces**. `--query-compute-apps` returns host PIDs, which
+    do not exist inside a container, so the process list is either empty or full of
+    `[Not Found]` process names. And the console itself runs in a container (`gpus: all`
+    in docker-compose.yml is only there to inject nvidia-smi and the driver libraries),
+    so under a production deployment use from outside the platform cannot be seen at
+    all -- the platform hands out a card that is already full as idle, the job goes OOM
+    the moment it starts, and the symptom is "there are four idle cards, why did it
+    insist on the full one".
+
+    Memory used is a **per-device** counter, unrelated to processes and identical inside
+    a container and out, so it has no such blind spot. The cost is that it cannot say
+    who is holding the card, which is why the two probes are unioned rather than one
+    replacing the other.
+    """
+    out = await _smi("--query-gpu=index,memory.used", "--format=csv,noheader,nounits")
+    return parse_gpu_memory_busy(out) if out is not None else None
+
+
+#: The columns the GPU health probe asks for. A field can be missing depending on the
+#: driver version (its value reads [N/A]), so parsing tolerates that per column.
+_HEALTH_QUERY = (
+    "index,retired_pages.pending,ecc.errors.uncorrected.volatile.total,"
+    "clocks_throttle_reasons.hw_slowdown"
+)
+
+
+def parse_gpu_health(text: str) -> dict[int, str]:
+    """nvidia-smi health query output -> {card index: why it is unhealthy}.
+
+    Only three signals count, all of them **certainly grounds to stop using the card**,
+    so that normal variation is not read as a fault:
+      - retired_pages.pending=Yes: memory pages are waiting to be retired, which needs
+        a GPU reset to take effect
+      - uncorrected ECC errors > 0: memory is already producing errors, so the training
+        result cannot be trusted
+      - hw_slowdown=Active: hardware throttling (heat or power), which makes training
+        silently slower
+
+    This kind of slow-or-broken **raises nothing** -- the job runs as usual, it is only
+    the result that is wrong or the speed that is down. Same family of failure as
+    "scattered PCIe cards make all-reduce slow": the hardest kind to diagnose.
+    """
+    out: dict[int, str] = {}
+    for line in text.splitlines():
+        parts = [p.strip() for p in line.split(",")]
+        if len(parts) < 4 or not parts[0].isdigit():
+            continue
+        idx = int(parts[0])
+        reasons = []
+        if parts[1].lower() == "yes":
+            reasons.append("pending retired memory pages (needs a GPU reset)")
+        if parts[2].isdigit() and int(parts[2]) > 0:
+            reasons.append(f"{parts[2]} uncorrected ECC errors")
+        if parts[3].lower() == "active":
+            reasons.append("hardware slowdown (heat or power)")
+        if reasons:
+            out[idx] = ", ".join(reasons)
+    return out
+
+
+async def _nvidia_smi_unhealthy() -> dict[int, str]:
+    """Which cards are unhealthy. Empty when it cannot be asked -- a health check that
+    fails only degrades to "no check", it does not over-subscribe the way a failed
+    occupancy probe does, so "unknown" and "none" need not be told apart here."""
+    out = await _smi(f"--query-gpu={_HEALTH_QUERY}", "--format=csv,noheader")
+    return parse_gpu_health(out) if out else {}
+
+
+async def detect_gpu_count() -> int:
+    """Physical card count on this machine. 0 when it cannot be probed, and the caller
+    falls back to the configured value."""
+    out = await _smi("--query-gpu=index", "--format=csv,noheader")
+    if not out:
+        return 0
+    return len([x for x in out.splitlines() if x.strip()])
+
+
+async def detect_gpu_name() -> str:
+    """NVML name of the first card (e.g. "NVIDIA H200"). Empty when it cannot be probed.
+
+    On a single-machine backend every card is the same model (one node), so the first
+    one stands for the machine.
+    """
+    out = await _smi("--query-gpu=name", "--format=csv,noheader")
+    return next((x.strip() for x in (out or "").splitlines() if x.strip()), "")
+
+
+def pick_gpus(free: list[int], count: int) -> list[int]:
+    """Pick `count` cards out of the free ones, **preferring consecutive indices**.
+
+    Why consecutive matters: jobs coming and going fragment the free set into shapes
+    like {0,3,5,7}. On a fully NVSwitch-connected machine (H200 SXM) it makes no
+    difference which cards are picked; on a PCIe machine, bandwidth between cards under
+    the same PCIe switch is markedly higher, and a scattered pick makes all-reduce
+    slower -- a slowness that raises nothing and leaves people believing training is
+    simply this slow.
+
+    When no consecutive window exists, fall back to taking them in order: running beats
+    running fast.
+    """
+    if count <= 0:
+        return []
+    ordered = sorted(free)
+    if len(ordered) < count:
+        raise NoCapacity(f"not enough free cards: need {count}, available {len(ordered)}")
+
+    for start in range(len(ordered) - count + 1):
+        window = ordered[start:start + count]
+        if window[-1] - window[0] == count - 1:      # consecutive
+            return window
+    return ordered[:count]
+
+
+#: Key and lease for the cross-process allocation lock. The lease has to cover "read
+#: occupancy + docker create/start" -- a few seconds when the image is already there,
+#: so 120s leaves plenty of room. While the lock is held a background task renews it
+#: every third of the lease, so genuinely slow work is not overtaken, and a process
+#: that crashes releases it after at most 120s.
+_ALLOC_LOCK_KEY = "gpu-alloc:leader"
+_ALLOC_LEASE_MS = 120_000
+_ALLOC_RETRY_S = 0.25
+
+
+class GpuAllocator:
+    """Hand physical cards to jobs.
+
+    Concurrency: the dequeue loop is serial, but the path where the API submits
+    directly runs alongside it. Allocation has to be one atomic "read occupancy -> pick
+    cards -> start the container", or two jobs pick the same cards. The lock lives on
+    the allocator rather than on callers, because the cost of one caller forgetting it
+    (silently stealing cards) is far worse than one extra wait.
+
+    ★ Two layers of lock, because concurrency has two sources:
+        In-process (asyncio.Lock) -- between direct API submissions inside the web
+          process, and between the in-process worker's dequeue loop and those.
+        Cross-process (Redis) -- with `TUNEPLANE_INPROCESS_WORKERS=0` the dequeue loop runs
+          inside `python -m server.worker --role scheduler`, which builds its own
+          LocalExecutor, its own GpuAllocator and its own asyncio.Lock; `uvicorn
+          --workers N` is the same story. With only the in-process lock those two
+          exclude each other not at all, and the double-allocation window is the few
+          seconds between occupancy() and docker run -- the failure this module exists
+          to prevent.
+      Without Redis configured the cross-process lock degrades to a no-op (NullRedis
+      always returns True), so the combination "local + several processes + no Redis"
+      is refused outright at startup; see server/bootstrap.py:check_allocation_safety.
+    """
+
+    def __init__(self, runtime, settings, *, redis=None, series_resolver=None):
+        """series_resolver: (configured profile, driver's card name) -> series id.
+
+        Injected because the hardware registry is the console's. Absent one --
+        which is every node -- the raw names are reported and the console maps
+        them at registration.
+        """
+        self.runtime = runtime
+        self.settings = settings
+        self.redis = redis
+        self._series_resolver = series_resolver
+        self._lock = asyncio.Lock()
+        self._total: Optional[int] = None
+        self._series: Optional[str] = None
+
+    async def total_gpus(self) -> int:
+        """Physical card count. An explicit setting wins: probing can fail when
+        the console itself runs inside a container.
+
+        A failed probe is not cached. The guard used to be `if self._total is
+        None`, but `configured or await detect_gpu_count()` stores whatever the
+        probe returned, and a failure returns 0 -- which is not None, so it was
+        cached for the lifetime of the process. One transient miss (nvidia-smi
+        briefly unavailable, or the 10s `_smi` timeout firing under load) then
+        pinned the node at zero cards forever: `occupancy().free` is empty so
+        every launch raises NoCapacity, and `cluster_gpus()` reports total=0 so
+        `series_total()` returns None and the capacity gate silently stops
+        checking anything. Only a restart recovered it.
+        """
+        if self._total is None or self._total <= 0:
+            configured = int(getattr(self.settings, "local_gpu_count", 0) or 0)
+            self._total = configured or await detect_gpu_count()
+        return self._total
+
+    async def series_id(self) -> str:
+        """Which series this machine's cards are (the hardware registry's series id);
+        empty when they cannot be identified.
+
+        The deployment's configured profile answers first -- an operator may have
+        classified them deliberately -- and with none configured the driver is asked for
+        the card name and the registry looked up. A series is a **hardware fact**, and
+        something that can be asked for directly should not depend on an operator
+        remembering a setting: a missing one does not cost "one group fewer". The
+        capacity view is organised by series, so cards that cannot be classified vanish
+        from the total as a batch and the page shows numbers that contradict themselves,
+        like "2 used / 0 total".
+
+        The result is cached: a series does not change over the life of the process,
+        while the capacity view is polled by the page.
+        An *unresolved* series is not cached, though. The guard used to be
+        `if self._series is None`, while the expression ends in `or ""` -- and
+        `""` is not None, so a single failed `detect_gpu_name()` pinned the node
+        as unclassifiable for the life of the process, producing exactly the
+        "used 2 / of 0" contradiction this docstring warns about.
+        """
+        if not self._series:
+            # Resolved by whoever knows the hardware registry. On a node nobody
+            # does -- the registry is the console's, editable in its admin page
+            # -- so the node reports the raw names and the console maps them.
+            # A node that guessed would be a second answer to a question one
+            # place already owns, and the two would drift the first time an
+            # administrator added a card.
+            profile = getattr(self.settings, "cluster_profile", "") or ""
+            resolve = self._series_resolver or _raw_series
+            # The configured profile answers first, and answering means the
+            # driver is never asked. An operator who declared these cards as a
+            # series did so deliberately -- sometimes to put a batch under a
+            # different one than the hardware suggests -- and probing anyway
+            # would be both slower and, on the machine, a contradiction.
+            self._series = resolve(profile, "") or ""
+            if not self._series:
+                self._series = resolve("", await detect_gpu_name()) or ""
+        return self._series
+
+    async def occupancy(self) -> GpuOccupancy:
+        """Occupancy now. Read afresh every time, never cached -- a stale cache means
+        over-subscription."""
+        total = await self.total_gpus()
+        states = await self.runtime.ps()
+
+        by_job: dict[int, str] = {}
+        for st in states:
+            # Only a live container holds cards. An exited one is still around waiting
+            # to be reclaimed, but it gave its GPU memory back long ago.
+            if st.status not in ("running", "created", "paused"):
+                continue
+            for idx in st.gpus:
+                if 0 <= idx < total:
+                    by_job[idx] = st.run_id or st.name
+
+        external: set[int] = set()
+        probe_ok = True
+        if getattr(self.settings, "local_check_external_gpus", True):
+            # Union of the two probes: the process list attributes use to somebody, and
+            # memory used is readable from inside a container too. Without the latter
+            # the whole check is off whenever the console runs in a container (see
+            # _nvidia_smi_memory_busy).
+            by_proc, by_mem = await asyncio.gather(
+                _nvidia_smi_busy(), _nvidia_smi_memory_busy()
+            )
+            probe_ok = by_mem is not None
+            seen = by_proc | (by_mem or set())
+            external = {i for i in seen if i not in by_job and i < total}
+
+        unhealthy: dict[int, str] = {}
+        if getattr(self.settings, "local_check_gpu_health", True):
+            unhealthy = {
+                i: reason
+                for i, reason in (await _nvidia_smi_unhealthy()).items()
+                # A card a platform job already holds is not deducted twice (it was
+                # never free), but the health problem still shows up in what
+                # health() / cluster_gpus report.
+                if i not in by_job and i < total
+            }
+
+        return GpuOccupancy(
+            total=total,
+            by_job=by_job,
+            external=external,
+            unhealthy=unhealthy,
+            external_probe_ok=probe_ok,
+        )
+
+    def _pick(self, run_id: str, count: int, occ: GpuOccupancy) -> list[int]:
+        """Pick cards, with the lock held and occupancy already read. Idempotent: the
+        same run_id gets back the cards it already holds."""
+        if existing := sorted(i for i, rid in occ.by_job.items() if rid == run_id):
+            log.info("run %s already holds cards %s, treating as idempotent", run_id, existing)
+            return existing
+
+        if count > occ.total:
+            # This job can never be scheduled, so it must not sit waiting in the queue.
+            raise NoCapacity(
+                f"the job needs {count} cards and this machine has {occ.total} -- "
+                f"no amount of waiting will place it"
+            )
+        try:
+            return pick_gpus(occ.free, count)
+        except NoCapacity:
+            raise NoCapacity(f"need {count} cards, but {occ.explain()}") from None
+
+    @contextlib.asynccontextmanager
+    async def _exclusive(self):
+        """The allocation critical section: the in-process lock, plus the cross-process
+        one when Redis is configured.
+
+        When Redis is unavailable `try_acquire` returns False and this retries forever
+        rather than letting the caller through -- allocation is an operation where
+        letting through means over-subscribing, so waiting for Redis to come back beats
+        handing out cards while unable to see the other processes. The caller (the
+        dequeue loop) already has the shape of retrying every 5s.
+        """
+        async with self._lock:
+            redis = self.redis
+            if redis is None or not getattr(redis, "enabled", False):
+                yield
+                return
+
+            token = _lock_token()
+            while not await redis.try_acquire(_ALLOC_LOCK_KEY, token, _ALLOC_LEASE_MS):
+                await asyncio.sleep(_ALLOC_RETRY_S)
+
+            async def _renew() -> None:
+                while True:
+                    await asyncio.sleep(_ALLOC_LEASE_MS / 3000.0)
+                    if not await redis.renew(_ALLOC_LOCK_KEY, token, _ALLOC_LEASE_MS):
+                        # The lease is no longer ours (Redis restarted, or somebody
+                        # cleared it by hand). Let this allocation finish, but record
+                        # it -- this is the one window where a double allocation is
+                        # possible.
+                        log.error(
+                            "lost the GPU allocation lock lease (key=%s); "
+                            "this allocation has no cross-process protection",
+                            _ALLOC_LOCK_KEY,
+                        )
+                        return
+
+            renewer = asyncio.create_task(_renew())
+            try:
+                yield
+            finally:
+                renewer.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await renewer
+                await redis.release(_ALLOC_LOCK_KEY, token)
+
+    async def allocate(self, run_id: str, count: int) -> list[int]:
+        """Pick cards without starting a container -- for dry runs and other paths that
+        take nothing.
+
+        ⚠ A real launch must go through allocate_and_run: this method releases the lock
+        as it returns, and the truth about occupancy lives in container labels, so
+        between "cards picked" and "container running" another allocation still sees
+        them as free.
+        """
+        if count <= 0:
+            return []
+        async with self._exclusive():
+            return self._pick(run_id, count, await self.occupancy())
+
+    async def allocate_and_run(self, run_id: str, count: int, run):
+        """Pick cards and start the container **under the same lock**, atomically;
+        returns (gpus, whatever run returned).
+
+        The lock has to cover starting the container: the occupancy view is rebuilt from
+        container labels through `docker ps`, so until the container is actually running
+        those cards still look free to everyone else -- and the dequeue loop running
+        alongside a direct API submission then gives the same cards to two jobs, which
+        is the failure this module exists to prevent. The cost is that container
+        creation on this machine is serialised; the local backend is one machine with a
+        low submission rate, so that is acceptable.
+
+        run: async callable(gpus: list[int]) -> T, called exactly once, inside the lock.
+        """
+        if count <= 0:
+            return [], await run([])
+        async with self._exclusive():
+            gpus = self._pick(run_id, count, await self.occupancy())
+            return gpus, await run(gpus)
+
+    async def release(self, run_id: str) -> None:
+        """Release explicitly.
+
+        The normal path **does not need this** -- once a container stops, `ps` no longer
+        reports it holding cards and the occupancy view converges on its own. The method
+        is here as a fallback for process mode, where the state file has to be cleared
+        by hand, and for tests.
+        """
+        for st in await self.runtime.ps():
+            if st.run_id != run_id:
+                continue
+            # Reclaim only what has exited. A running container must not be removed --
+            # that is killing somebody's training run without telling them.
+            if st.status not in ("running", "created", "paused"):
+                await self.runtime.remove(st.name)
+            return