tuneplane-node 0.3.15__py3-none-any.whl

tuneplane_node/daemon.py

@@ -0,0 +1,497 @@
+"""The node daemon: a narrow job API on a GPU node.
+
+What it is
+──────────────────────────────────────────────────────────────────────────────
+The node-side component of the node backend (kubelet is the analogue under K8s): the
+console calls it over HTTP to launch, observe and stop job containers. **The daemon is
+control-plane infrastructure and runs no training** -- a job is still a per-job
+container, with exactly the local backend's semantics (one throwaway container per
+job, destroyed when it finishes).
+
+Reuse rather than rewrite
+──────────────────────────────────────────────────────────────────────────────
+Every core mechanism inside the node is the local backend's implementation:
+  container runtime  node/runtime.py (the docker / podman CLI wrapper)
+  GPU allocation     node/allocator.py (allocation lock, labels as the truth,
+                     health probing)
+  image prefetch     the same explicit pull on its own long timeout as LocalExecutor
+
+So restarting a node loses no state at all: the truth about occupancy is in the
+container labels and one `docker ps` rebuilds it -- the property the local backend's
+design bought in the first place.
+
+Security model
+──────────────────────────────────────────────────────────────────────────────
+Only the job verbs are exposed (launch/observe/stop/logs/reap), never arbitrary
+container operations; a bearer token is mandatory (without one the daemon refuses to
+start rather than running open). Against the alternative -- the console holding each
+node's docker socket, which is equivalent to root -- this narrows the privilege
+surface.
+
+Asynchronous semantics of an image pull
+──────────────────────────────────────────────────────────────────────────────
+A first pull of tens of GB takes tens of minutes and cannot hang off a single HTTP
+request. When the image is not on the machine, launch returns phase=pulling at once
+and a background task finishes the pull, then allocates GPUs and starts the container;
+observe on the console side reports PENDING with the pull explained meanwhile. A
+failed pull becomes an explicit failed state carrying the reason, rather than
+disappearing quietly. If the daemon restarts mid-pull the pending record is gone and
+no container exists, so reconciliation converges on STOPPED -- which the docs state,
+and resubmitting is the answer.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import secrets
+import shutil
+import time
+from dataclasses import dataclass, field
+from weakref import WeakValueDictionary
+
+from fastapi import Depends, FastAPI, HTTPException, Request
+
+from tuneplane_node.allocator import GpuAllocator, NoCapacity
+from tuneplane_node.runtime import LABEL_GPUS, ContainerState, detect_runtime
+from tuneplane_node.runtime import age_seconds as _age_seconds
+from tuneplane_node.wire import PROTOCOL, spec_from_dict, state_to_dict
+
+log = logging.getLogger(__name__)
+
+
+#: How long a failed pending record is kept (seconds). It has to outlive a round of
+#: reconciliation -- the console must see the failure reason before it can record it in
+#: the ledger -- and after that it is nothing but memory held for no reason.
+_FAILED_PENDING_TTL_S = 1800.0
+
+
+@dataclass
+class PendingLaunch:
+    """The explicit state of one asynchronous launch, while its image is pulling."""
+
+    run_id: str
+    image: str
+    #: ★ How many cards this job has been promised. It has to be recorded: the pull
+    #: takes tens of minutes, no container exists during it, and the truth about
+    #: occupancy is the container labels -- so occupancy() cannot see the job at all.
+    #: Without this field health reports the promised cards as free, the control
+    #: plane's _place puts later jobs on the same node, and by the time the image is
+    #: pulled the cards are long gone and the job ends in "please resubmit". Queueing
+    #: for tens of minutes and then running nothing is the worst kind of failure.
+    requested_gpus: int = 0
+    phase: str = "pulling"          # pulling | starting | failed
+    error: str = ""
+    started_at: float = field(default_factory=time.time)
+
+    @property
+    def reserves_gpus(self) -> bool:
+        """Still waiting for cards; a failed launch holds none."""
+        return self.phase != "failed"
+
+
+class NodeState:
+    """The daemon's in-process state. runtime and allocator are injectable, for test doubles."""
+
+    def __init__(self, settings, *, runtime=None, allocator=None):
+        self.settings = settings
+        self._runtime = runtime
+        self._allocator = allocator
+        self.pending: dict[str, PendingLaunch] = {}
+        #: Strong references to the background pull tasks. asyncio holds only a weak
+        #: one, so a create_task whose result is dropped can be garbage-collected
+        #: mid-pull -- and the job then sits in pulling forever, with no exception and
+        #: nothing in the log.
+        self._tasks: set = set()
+        #: Background launches, by container ref, so a stop can find the one it
+        #: is stopping. Without this a stop during a pull popped the pending
+        #: record and returned ok, while the task went on to allocate GPUs and
+        #: start the container minutes later -- the console had already released
+        #: the quota, so nothing ever reclaimed those cards.
+        self._launches: dict[str, "asyncio.Task"] = {}
+        #: Refs whose launch was cancelled. Checked at both await boundaries of
+        #: `_pull_then_start`, because cancelling a task only helps if it is
+        #: suspended: a cancel that lands while the container is being created
+        #: has to be answered by tearing that container down instead.
+        self.cancelled: set[str] = set()
+        self._control_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary()
+
+    def control_lock(self, ref: str) -> asyncio.Lock:
+        """Serialize create/start with stop/remove for one container.
+
+        Holders and waiters keep the lock alive; completed container identities
+        do not accumulate in a daemon that may run for months.
+        """
+        lock = self._control_locks.get(ref)
+        if lock is None:
+            lock = asyncio.Lock()
+            self._control_locks[ref] = lock
+        return lock
+
+    def spawn(self, coro, *, ref: str = "") -> None:
+        """Start a background task and hold a reference to it until it finishes."""
+        task = asyncio.get_running_loop().create_task(coro)
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+        if ref:
+            self._launches[ref] = task
+            task.add_done_callback(lambda _t, r=ref: self._launches.pop(r, None))
+
+    def begin_launch(self, ref: str) -> None:
+        """A fresh launch for `ref` clears any tombstone an earlier one left."""
+        self.cancelled.discard(ref)
+
+    def cancel_launch(self, ref: str) -> None:
+        """Stop a launch in flight, whatever stage it has reached."""
+        self.cancelled.add(ref)
+        task = self._launches.pop(ref, None)
+        if task is not None and not task.done():
+            task.cancel()
+        self.pending.pop(ref, None)
+
+    def is_cancelled(self, ref: str) -> bool:
+        return ref in self.cancelled
+
+    def reserved_gpus(self) -> int:
+        """Cards promised to pending jobs whose container has not started yet."""
+        self.expire_failed_pending()
+        return sum(p.requested_gpus for p in self.pending.values() if p.reserves_gpus)
+
+    def expire_failed_pending(self, now: float | None = None) -> int:
+        """Drop failed records that have expired.
+
+        On the normal path the console reconciles to a terminal state and then calls
+        remove/stop, which takes the record with it. But a FAILED training job stays in
+        the ledger so it can be retried automatically, remove never comes, and the
+        records pile up in the daemon's memory. Hence a TTL, rather than relying on the
+        caller to collect them.
+        """
+        now = now if now is not None else time.time()
+        stale = [
+            name for name, p in self.pending.items()
+            if p.phase == "failed" and (now - p.started_at) > _FAILED_PENDING_TTL_S
+        ]
+        for name in stale:
+            self.pending.pop(name, None)
+        return len(stale)
+
+    @property
+    def runtime(self):
+        if self._runtime is None:
+            self._runtime = detect_runtime(
+                getattr(self.settings, "local_runtime", "auto"),
+                state_dir=self.settings.storage.state_root / "local",
+                timeout=float(getattr(self.settings, "local_cli_timeout", 60.0)),
+            )
+        return self._runtime
+
+    @property
+    def allocator(self) -> GpuAllocator:
+        if self._allocator is None:
+            self._allocator = GpuAllocator(self.runtime, self.settings)
+        return self._allocator
+
+
+def _node_series(settings) -> str:
+    """What this node was configured as, reported as-is.
+
+    Not mapped to a hardware series here. The series registry belongs to the
+    console -- it is editable in its admin page -- so a node that mapped would be
+    a second answer to a question one place already owns, and the two would drift
+    the first time an administrator added a card. The node reports facts (the
+    configured profile, and the card name the driver gives); the console
+    interprets them.
+    """
+    return str(getattr(settings, "cluster_profile", "") or "").strip().lower()
+
+
+def create_node_app(settings, *, runtime=None, allocator=None) -> FastAPI:
+    token = str(getattr(settings, "node_token", "") or "").strip()
+    if not token:
+        # Fail closed: with no token, the node's ability to launch containers is open to
+        # anything on the private network.
+        raise ValueError("the node daemon requires TUNEPLANE_NODE_TOKEN to be set explicitly")
+
+    state = NodeState(settings, runtime=runtime, allocator=allocator)
+    app = FastAPI(title="tuneplane-node", docs_url=None, redoc_url=None, openapi_url=None)
+    app.state.agent = state
+
+    expected_auth = f"Bearer {token}"
+
+    async def _auth(request: Request) -> None:
+        header = request.headers.get("authorization", "")
+        # Constant-time comparison: `!=` returns at the first differing byte and the
+        # token has a fixed length, which makes probing it byte by byte practical from
+        # inside the private network.
+        if not secrets.compare_digest(header, expected_auth):
+            raise HTTPException(401, "invalid node token")
+        wire = request.headers.get("x-tuneplane-node-protocol", "")
+        # The protocol header is required. It used to read `if wire and ...`,
+        # which let a request without one straight through -- so "both sides
+        # check on every request" actually meant "checked when sent", and a
+        # console that sent none passed silently and was handled by whatever
+        # each side happened to assume. That is the exact case fail-closed
+        # exists to prevent.
+        if not wire:
+            raise HTTPException(
+                400,
+                f"missing X-TunePlane-Node-Protocol header (this node speaks wire v{PROTOCOL}); "
+                "upgrade the console, or check that no proxy strips the header",
+            )
+        if wire != str(PROTOCOL):
+            raise HTTPException(409, f"wire protocol mismatch: console={wire}, node={PROTOCOL}")
+
+    auth = Depends(_auth)
+
+    # ── Launch ──────────────────────────────────────────────────────────────
+
+    async def _start_container(spec, run_id: str, requested_gpus: int) -> dict:
+        async def _run(gpus: list[int]) -> str:
+            # The node writes the pick back itself: GPU passthrough follows the node's
+            # own setting (a simulated node turns it off) while the allocation label is
+            # written either way -- the same semantics as the local backend's
+            # simulation mode.
+            passthrough = bool(getattr(settings, "local_gpu_passthrough", True))
+            spec.gpus = gpus if passthrough else []
+            spec.labels[LABEL_GPUS] = ",".join(str(i) for i in gpus)
+            return await state.runtime.run(spec)
+
+        gpus, cid = await state.allocator.allocate_and_run(run_id, requested_gpus, _run)
+        return {"phase": "running", "container_id": cid[:12], "gpus": gpus}
+
+    async def _pull_then_start(name: str, spec, run_id: str, requested_gpus: int) -> None:
+        # `.get`, not `[...]`: a stop landing before this task's first slice has
+        # already popped the record, and a KeyError here would be an unhandled
+        # exception in a background task rather than the no-op it should be.
+        pending = state.pending.get(name)
+        if pending is None or state.is_cancelled(name):
+            return
+        try:
+            timeout = float(getattr(settings, "local_image_pull_timeout_s", 3600.0))
+            await state.runtime.pull(spec.image, timeout=timeout)
+            async with state.control_lock(name):
+                # The pull is documented as taking tens of minutes. Anything can have
+                # happened in that window, and the thing that usually did is a stop.
+                if state.is_cancelled(name) or state.pending.get(name) is not pending:
+                    return
+                pending.phase = "starting"
+                await _start_container(spec, run_id, requested_gpus)
+                if state.is_cancelled(name):
+                    # The cancel landed *during* create+start. The container exists
+                    # and holds real cards, and the caller was told the stop
+                    # succeeded, so tearing it down here is the only thing that can.
+                    await state.runtime.stop(name, timeout=5)
+                    await state.runtime.remove(name)
+                    return
+                state.pending.pop(name, None)
+        except asyncio.CancelledError:
+            # A cancel while suspended. The container was never created, so
+            # there is nothing to tear down -- but the record must not be left
+            # behind reserving cards in `reserved_gpus()`.
+            state.pending.pop(name, None)
+            raise
+        except NoCapacity as exc:
+            # A concurrent job took the cards during the pull: fail explicitly and say
+            # what to do next rather than queueing on the node -- queueing belongs to
+            # the console's scheduler, and a second queue must not grow here.
+            pending.phase = "failed"
+            pending.error = f"the GPUs were taken during the image pull ({exc}); submit again"
+        except Exception as exc:  # noqa: BLE001
+            pending.phase = "failed"
+            pending.error = f"image pull or launch failed: {exc}"
+            log.exception("asynchronous launch of %s failed", name)
+
+    @app.post("/node/v1/launch", dependencies=[auth])
+    async def launch(body: dict):
+        run_id = str(body.get("run_id") or "").strip()
+        requested_gpus = int(body.get("requested_gpus") or 0)
+        if not run_id or "container" not in body:
+            raise HTTPException(422, "launch requires run_id and container")
+        spec = spec_from_dict(body["container"])
+
+        async with state.control_lock(spec.name):
+            # Repeated launches observe the container created by their predecessor.
+            existing = await state.runtime.inspect(spec.name)
+            if existing.exists and existing.status in ("running", "created", "paused"):
+                return {"phase": "running", "idempotent": True, "gpus": existing.gpus}
+            if existing.exists:
+                await state.runtime.remove(spec.name)
+            if (pending := state.pending.get(spec.name)) and pending.phase != "failed":
+                return {"phase": pending.phase, "idempotent": True, "image": pending.image}
+
+            state.begin_launch(spec.name)
+            if not await state.runtime.image_exists(spec.image):
+                # Reserve capacity while the image is pulled in the background.
+                state.pending[spec.name] = PendingLaunch(
+                    run_id=run_id, image=spec.image, requested_gpus=requested_gpus,
+                )
+                state.spawn(
+                    _pull_then_start(spec.name, spec, run_id, requested_gpus), ref=spec.name,
+                )
+                return {"phase": "pulling", "image": spec.image,
+                        "requested_gpus": requested_gpus}
+            try:
+                return await _start_container(spec, run_id, requested_gpus)
+            except NoCapacity as exc:
+                raise HTTPException(409, str(exc)) from exc
+
+    # ── Observe ─────────────────────────────────────────────────────────────
+
+    @app.post("/node/v1/observe", dependencies=[auth])
+    async def observe(body: dict):
+        refs = [str(x) for x in (body.get("refs") or [])]
+        states = {st.name: st for st in await state.runtime.ps()}
+        out: dict[str, dict] = {}
+        for ref in refs:
+            if st := states.get(ref):
+                out[ref] = state_to_dict(st)
+                continue
+            if pending := state.pending.get(ref):
+                if pending.phase == "failed":
+                    out[ref] = {
+                        **state_to_dict(ContainerState(name=ref, exists=True, status="dead")),
+                        "pending_error": pending.error,
+                    }
+                else:
+                    # Pulling or starting: presented to the ledger as created (-> PENDING),
+                    # never as gone.
+                    out[ref] = {
+                        **state_to_dict(ContainerState(name=ref, exists=True, status="created")),
+                        "pending_phase": pending.phase,
+                        "pending_image": pending.image,
+                        # How long it has been waiting. The control plane uses it to say
+                        # "still pulling, 12 minutes in" instead of leaving somebody
+                        # staring at a motionless PENDING, guessing whether it is stuck.
+                        "pending_elapsed_s": round(time.time() - pending.started_at, 1),
+                        "pending_gpus": pending.requested_gpus,
+                    }
+                continue
+            out[ref] = state_to_dict(ContainerState(name=ref, exists=False))
+        return {"states": out}
+
+    # ── Control and logs ────────────────────────────────────────────────────
+
+    @app.post("/node/v1/stop", dependencies=[auth])
+    async def stop(body: dict):
+        ref = str(body.get("ref") or "")
+        # Cancelling the pending record is not enough: the background pull task
+        # has to be stopped too. Popping the record alone left it running, so it
+        # finished the pull, allocated the GPUs and started the container --
+        # after the console had already released the quota for a run it was told
+        # had stopped, which is how those cards ended up held by nothing.
+        async with state.control_lock(ref):
+            state.cancel_launch(ref)
+            ok = await state.runtime.stop(
+                ref, timeout=int(body.get("timeout") or getattr(settings, "local_stop_timeout", 30))
+            )
+            return {"ok": ok}
+
+    @app.post("/node/v1/remove", dependencies=[auth])
+    async def remove(body: dict):
+        ref = str(body.get("ref") or "")
+        async with state.control_lock(ref):
+            state.cancel_launch(ref)
+            return {"ok": await state.runtime.remove(ref)}
+
+    @app.get("/node/v1/logs", dependencies=[auth])
+    async def logs(ref: str, tail: int = 2000):
+        return {"logs": await state.runtime.logs(ref, tail_lines=tail)}
+
+    @app.post("/node/v1/reap", dependencies=[auth])
+    async def reap(body: dict):
+        ttl = int(body.get("max_age_s") or 0) or int(
+            getattr(settings, "local_container_ttl_s", 600)
+        )
+        now = time.time()
+        removed = 0
+        for st in await state.runtime.ps():
+            if st.status in ("running", "paused"):
+                continue
+            if st.status == "created":
+                # Wreckage from a successful create whose start failed. It never
+                # reaches `exited` on its own, yet it carries LABEL_GPUS and so
+                # `occupancy()` counts it as holding its cards -- leaving it is a
+                # permanent GPU leak. `LocalExecutor.reap` has handled this since
+                # it was found there; the node backend skipped every `created`
+                # container unconditionally.
+                #
+                # Only one that never started and is past its TTL: a healthy
+                # launch is `created` for milliseconds, because
+                # `allocate_and_run` holds the lock across create and start.
+                if not st.never_started:
+                    continue
+                if ttl > 0 and _age_seconds(st.created_at, now) < ttl:
+                    continue
+            elif ttl > 0 and _age_seconds(st.finished_at, now) < ttl:
+                continue
+            if await state.runtime.remove(st.name):
+                removed += 1
+        return {"removed": removed}
+
+    # ── Health and capacity ─────────────────────────────────────────────────
+
+    @app.get("/node/v1/health", dependencies=[auth])
+    async def health():
+        base = await state.runtime.health()
+        if not base.get("ok"):
+            return {"ok": False, "runtime": base, "protocol": PROTOCOL}
+        try:
+            occ = await state.allocator.occupancy()
+        except Exception as exc:  # noqa: BLE001
+            return {"ok": False, "runtime": base, "protocol": PROTOCOL,
+                    "detail": f"GPU probe failed: {exc}"}
+        # The node's view of the shared storage root. The console reports its
+        # own; both should be the same filesystem, and a node that disagrees is
+        # exactly the misconfiguration worth surfacing here.
+        disk = None
+        root = settings.storage.root
+        if settings.storage_configured and root.exists():
+            try:
+                du = shutil.disk_usage(root)
+                disk = {"path": str(root), "used_pct": round(du.used / du.total * 100.0, 1),
+                        "free_gib": round(du.free / 1024**3, 1)}
+            except OSError:
+                disk = None
+        # ★ free = physically free minus the cards promised to jobs still pulling. The
+        #   control plane's _place reads nothing else, so without the subtraction it
+        #   promises the same cards twice and the job waits tens of minutes to find out
+        #   it cannot have them.
+        reserved = state.reserved_gpus()
+        free_gpus = occ.free[reserved:] if reserved else occ.free
+        return {
+            "ok": True,
+            "protocol": PROTOCOL,
+            "runtime": base,
+            # This node's series. Where nodes hold different cards, it is what lets the
+            # control plane place a job on the right ones and count its capacity in the
+            # right bucket -- without it every node is taken to hold whatever the
+            # console's own cluster_profile says, an H100 node's cards are reported as
+            # H200, and every control that works per series stops working.
+            "series": _node_series(settings),
+            "gpus_total": occ.total,
+            "gpus_free": max(0, len(occ.free) - reserved),
+            "gpus_reserved": reserved,
+            "free_gpus": free_gpus,
+            "external": sorted(occ.external),
+            # When detection is broken `external` is always empty, so the control plane
+            # has to be able to tell "nobody holds a card on this node" from "this node
+            # cannot see whether anybody does" -- otherwise an inflated free count is
+            # undiscoverable.
+            "external_probe_ok": occ.external_probe_ok,
+            "unhealthy": {str(k): v for k, v in occ.unhealthy.items()},
+            "by_job": {str(k): v for k, v in occ.by_job.items()},
+            "gpu_detail": occ.explain(),
+            "disk": disk,
+            "pending_pulls": {
+                name: {"image": p.image, "phase": p.phase, "error": p.error}
+                for name, p in state.pending.items()
+            },
+        }
+
+    return app
+
+
+def make_app() -> FastAPI:
+    """The uvicorn factory entry point, used by `tuneplane-node serve`."""
+    from tuneplane_node.settings import NodeSettings
+
+    return create_node_app(NodeSettings())

tuneplane_node/inventory.py

@@ -0,0 +1,163 @@
+"""What this machine has, collected in one place.
+
+**This is the tier-3 seam.** Everything the platform does with an inventory --
+refusing a shared-mount fleet, matching a series, deciding a node can hold a job
+-- works from the dataclass below, and only `collect()` touches hardware. A
+runner with no GPU therefore exercises registration, gating, drain and the whole
+console surface; one function stays unproven, and its size is known (CLAUDE.md,
+*Where a check runs*).
+
+It answers only what the console cannot see for itself and cannot be told
+truthfully by configuration: the driver, the container runtime, the cards, and
+whether the shared storage root is actually here.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+from dataclasses import asdict, dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class NodeInventory:
+    #: What the platform calls these cards, e.g. "h100". Empty when the driver
+    #: reports a card the series registry has never heard of.
+    accelerator_series: str = ""
+    #: What the driver calls them, e.g. "NVIDIA H100 80GB HBM3". Kept alongside
+    #: the series because an unmapped card is exactly the case where an operator
+    #: needs to see the real name to add it to the registry.
+    accelerator_name: str = ""
+    accelerator_count: int = 0
+    driver: str = ""
+    runtime: str = ""
+    #: Whether the Storage Root is present at the same absolute path the console
+    #: uses. A shared-mount Fleet refuses a node that answers False, because the
+    #: alternative is accepting it and failing at the first launch with an
+    #: unrelated-looking I/O error.
+    storage_root_visible: bool = False
+    disk_free_bytes: int = 0
+    cpu_count: int = 0
+    memory_bytes: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+async def probe_gpus() -> tuple[str, str, int]:
+    """(card name, driver version, card count), straight from the driver.
+
+    The one place in the platform that shells out to `nvidia-smi`, and the
+    reason this module is the tier-3 seam.
+
+    **Asking the driver rather than reading configuration is the point.** The
+    node-side series used to come from this machine's own `TUNEPLANE_CLUSTER_PROFILE`,
+    which is a value an operator sets and can forget: a mixed-GPU fleet whose
+    nodes were never configured reports no series at all, and every placement
+    rule that filters by series silently stops filtering. The machine knows what
+    cards it has; nothing else reliably does.
+
+    A machine with no `nvidia-smi` answers ("", "", 0), which the console reads
+    as unknown. That is a fact, and guessing from the container image would not
+    be.
+    """
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader",
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
+        )
+        out, _ = await asyncio.wait_for(proc.communicate(), timeout=10)
+    except (FileNotFoundError, OSError, asyncio.TimeoutError):
+        return "", "", 0
+    if proc.returncode != 0:
+        return "", "", 0
+    # One line per card. They are homogeneous on any machine the platform can
+    # place a pool on, so the first line names them all; the line count is the
+    # card count, which is also what the allocator sees.
+    lines = [ln.strip() for ln in (out or b"").decode(errors="replace").splitlines() if ln.strip()]
+    if not lines:
+        return "", "", 0
+    name, _, driver = lines[0].partition(",")
+    return name.strip(), driver.strip(), len(lines)
+
+
+def _memory_bytes() -> int:
+    try:
+        return int(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES"))
+    except (ValueError, OSError, AttributeError):
+        # Not every platform exposes these, and a missing figure is not worth
+        # failing a registration over.
+        return 0
+
+
+def _series_of(gpu_name: str, settings) -> str:
+    """What this node was configured as, reported as-is.
+
+    **Not mapped to a hardware series here.** The registry that knows which card
+    names belong to which series is the console's, editable in its admin page; a
+    node that mapped would be a second answer to a question one place already
+    owns, and the two would drift the first time an administrator added a card.
+
+    So the node reports two facts -- the profile an operator declared for this
+    machine, and the name the driver gives the cards -- and the console decides
+    what they mean. Both may be empty, and empty is honest: placement treats an
+    unknown series as "do not exclude this node" rather than pretending to know.
+    """
+    from tuneplane_node.daemon import _node_series
+
+    return _node_series(settings)
+
+
+async def collect(settings, *, runtime=None, allocator=None) -> NodeInventory:
+    """Ask this machine what it is. Never raises: a partial answer beats none.
+
+    A field the machine cannot answer stays at its zero value, which the console
+    reads as "unknown" rather than "none" -- the two matter for capacity but not
+    for whether the node may join, and inventing a number here would be worse
+    than an empty one.
+    """
+    from tuneplane_node.allocator import GpuAllocator
+    from tuneplane_node.runtime import detect_runtime
+
+    runtime = runtime or detect_runtime(settings)
+    allocator = allocator or GpuAllocator(settings, runtime=runtime)
+
+    runtime_name = ""
+    driver = ""
+    try:
+        health = await runtime.health()
+        runtime_name = f"{getattr(runtime, 'name', '')}://{health.get('version') or ''}".strip("/:")
+    except Exception:  # noqa: BLE001
+        runtime_name = str(getattr(runtime, "name", "") or "")
+
+    gpu_name, driver, probed = await probe_gpus()
+    count = probed
+    try:
+        # The allocator's figure wins when it has one: it is what actually
+        # decides placement on this node, and a disagreement between it and the
+        # driver is the allocator's to explain.
+        count = int((await allocator.occupancy()).total) or probed
+    except Exception:  # noqa: BLE001
+        count = probed
+
+    root = settings.storage.root
+    visible = bool(getattr(settings, "storage_configured", False)) and root.exists()
+    free = 0
+    if visible:
+        try:
+            free = int(shutil.disk_usage(root).free)
+        except OSError:
+            visible = False
+
+    return NodeInventory(
+        accelerator_series=_series_of(gpu_name, settings),
+        accelerator_name=gpu_name,
+        accelerator_count=count,
+        driver=driver,
+        runtime=runtime_name,
+        storage_root_visible=visible,
+        disk_free_bytes=free,
+        cpu_count=os.cpu_count() or 0,
+        memory_bytes=_memory_bytes(),
+    )