tuneplane-node 0.3.15__py3-none-any.whl

tuneplane_node/runtime.py

@@ -0,0 +1,611 @@
+"""Thin wrapper over a container runtime: docker / podman / bare process.
+
+Why the CLI rather than an SDK
+──────────────────────────────────────────────────────────────────────────────
+docker's Python SDK is synchronous and blocking, which does not fit this service's
+async stack (either wrap it in a thread pool or take a community fork); podman's
+HTTP API is a different interface again, so both would mean two clients to keep.
+
+Six actions are all that is used: run / inspect / logs / stop / rm / ps. podman's
+CLI is deliberately docker-compatible and the flags for those six are all but
+identical -- the CLI is the greatest common factor of the two, and
+`asyncio.create_subprocess_exec` is async without help.
+
+**argv lists throughout, never a shell.** A job's entrypoint comes from an
+experiment the user uploaded and its env holds user-controlled values; one place
+that joins either into a shell string is an injection surface.
+
+The trade-off between the three runtimes
+──────────────────────────────────────────────────────────────────────────────
+  docker    the default. Widest ecosystem, `--gpus` natively
+  podman    daemonless, can run rootless; GPUs through CDI
+            (`--device nvidia.com/gpu=N`)
+  process   ★ no isolation, only for a dev machine with no container runtime
+
+The cost of process mode has to be stated plainly: it falls back to "the process
+shares everything with the host", which is the problem containerisation was adopted
+to solve -- a crashed training process can leave GPU memory held, and jobs have no
+resource boundary between them. Never the default.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import shutil
+import signal
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional, Protocol, runtime_checkable
+
+log = logging.getLogger(__name__)
+
+#: Label prefix. Container labels are **the only truth about allocation** -- the
+#: console rebuilds "which cards are held by whom" from them after a restart, so
+#: this cannot live in memory alone.
+LABEL_RUN_ID = "tuneplane.run-id"
+LABEL_GPUS = "tuneplane.gpus"
+LABEL_USER = "tuneplane.user"
+LABEL_KIND = "tuneplane.kind"
+#: Which execution of that run this container is (Job.attempt on the console
+#: side). The name is a pure function of the run id, so it is reused by every
+#: later attempt; this label is the only thing on the container that tells them
+#: apart, and it is what a delayed cleanup checks before removing it.
+LABEL_ATTEMPT = "tuneplane.attempt"
+
+#: Container name prefix. A fixed prefix rather than a random name is what makes a
+#: repeated launch of the same run_id produce one job instead of two.
+NAME_PREFIX = "tuneplane-"
+
+
+def container_name(run_id: str) -> str:
+    """run_id -> container name. A docker name allows [a-zA-Z0-9][a-zA-Z0-9_.-]*."""
+    safe = "".join(ch if (ch.isalnum() or ch in "_.-") else "-" for ch in (run_id or ""))
+    return f"{NAME_PREFIX}{safe.strip('-') or 'job'}"
+
+
+@dataclass
+class ContainerSpec:
+    """Everything one container launch needs, already resolved: the runtime decides nothing."""
+
+    name: str
+    image: str
+    #: Entry argv. **Not a shell string** -- where shell semantics are wanted, write
+    #: ["bash", "-lc", "..."] explicitly, so the injection surface stays at the caller
+    #: instead of being spread through here.
+    command: list[str]
+    env: dict[str, str] = field(default_factory=dict)
+    #: Physical GPU indices. An empty list means no cards (export, eval and the like).
+    gpus: list[int] = field(default_factory=list)
+    #: (host path, path inside the container, read-only)
+    mounts: list[tuple[str, str, bool]] = field(default_factory=list)
+    labels: dict[str, str] = field(default_factory=dict)
+    workdir: str = ""
+    shm_size: str = ""
+    cpu_limit: str = ""
+    memory_limit: str = ""
+    #: Training reports its status back to the console, and sharing the host's network
+    #: is the least work for the single-machine, private-network case.
+    network: str = "host"
+    user: str = ""
+
+
+@dataclass
+class ContainerState:
+    """The one result shape inspect / ps return."""
+
+    name: str
+    exists: bool = False
+    #: The runtime's own status: created / running / exited / dead / paused ...
+    status: str = ""
+    exit_code: Optional[int] = None
+    labels: dict[str, str] = field(default_factory=dict)
+    created_at: str = ""
+    started_at: str = ""
+    finished_at: str = ""
+    #: Killed by the kernel OOM killer. Recorded on its own because its exit code is
+    #: 137 too, exactly like a `stop` that escalated to SIGKILL -- reading the exit code
+    #: alone reports "out of memory" as "the user stopped it", and what to do next is
+    #: the opposite in each case (lower the batch size vs. nothing at all).
+    oom_killed: bool = False
+
+    @property
+    def gpus(self) -> list[int]:
+        """Which cards this container holds, recovered from its labels."""
+        raw = self.labels.get(LABEL_GPUS, "")
+        return [int(x) for x in raw.split(",") if x.strip().isdigit()]
+
+    @property
+    def run_id(self) -> str:
+        return self.labels.get(LABEL_RUN_ID, "")
+
+    @property
+    def attempt(self) -> int:
+        """Which execution of that run this is. 0 when the label is absent,
+        which is what a container created before the label existed looks like."""
+        raw = self.labels.get(LABEL_ATTEMPT, "")
+        return int(raw) if raw.strip().lstrip("-").isdigit() else 0
+
+    @property
+    def never_started(self) -> bool:
+        """Created successfully but never once started.
+
+        docker reports a zero StartedAt (0001-01-01) for a container that never ran.
+        Such a container never reaches exited on its own, yet it carries LABEL_GPUS and
+        so counts as holding cards -- reclaim has to be able to recognise it.
+        """
+        return not self.started_at or self.started_at.startswith("0001-01-01")
+
+
+class RuntimeError_(RuntimeError):
+    """A runtime call failed. Carries stderr -- that is where docker puts the reason."""
+
+    def __init__(self, message: str, *, code: int = 0, stderr: str = ""):
+        super().__init__(message)
+        self.code = code
+        self.stderr = stderr
+
+
+@runtime_checkable
+class ContainerRuntime(Protocol):
+    name: str
+
+    async def run(self, spec: ContainerSpec) -> str: ...
+
+    async def inspect(self, name: str) -> ContainerState: ...
+
+    async def logs(self, name: str, *, tail_lines: int = 2000) -> str: ...
+
+    async def stop(self, name: str, *, timeout: int = 30) -> bool: ...
+
+    async def remove(self, name: str) -> bool: ...
+
+    async def ps(self) -> list[ContainerState]: ...
+
+    async def health(self) -> dict: ...
+
+    async def image_exists(self, image: str) -> bool: ...
+
+    async def pull(self, image: str, *, timeout: float) -> None: ...
+
+
+# ── CLI runtime (docker / podman share it) ───────────────────────────────────
+
+
+class CliRuntime:
+    """docker / podman. Their CLIs agree on these six actions; only the GPU flags differ."""
+
+    def __init__(self, binary: str, *, timeout: float = 60.0):
+        self.binary = binary
+        self.name = Path(binary).name
+        self.timeout = timeout
+
+    # ── The underlying call ─────────────────────────────────────────────────
+
+    async def _exec(
+        self, *args: str, check: bool = True, timeout: float | None = None
+    ) -> tuple[int, str, str]:
+        limit = timeout if timeout is not None else self.timeout
+        proc = await asyncio.create_subprocess_exec(
+            self.binary, *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        try:
+            out, err = await asyncio.wait_for(proc.communicate(), timeout=limit)
+        except asyncio.TimeoutError:
+            proc.kill()
+            raise RuntimeError_(f"{self.name} {args[0]} timed out after {limit}s") from None
+        code = proc.returncode or 0
+        stdout, stderr = out.decode(errors="replace"), err.decode(errors="replace")
+        if check and code != 0:
+            raise RuntimeError_(
+                f"{self.name} {args[0]} failed (exit {code}): {stderr.strip()[:400]}",
+                code=code, stderr=stderr,
+            )
+        return code, stdout, stderr
+
+    # ── GPU flags: the only real difference between docker and podman ───────
+
+    def _gpu_args(self, gpus: list[int]) -> list[str]:
+        if not gpus:
+            return []
+        ids = ",".join(str(i) for i in gpus)
+        if self.name == "podman":
+            # podman goes through CDI, one --device per card. Requires that
+            # `nvidia-ctk cdi generate` has already been run.
+            return [arg for i in gpus for arg in ("--device", f"nvidia.com/gpu={i}")]
+        # ★ The double quotes are load-bearing, not stray escaping: the docker CLI parses
+        #   the value of --gpus with a csv.Reader, so a bare `device=0,1` is split into
+        #   ["device=0", "1"] and every field without an `=` is read as a count -- one
+        #   DeviceRequest then carries both Count and DeviceIDs and the daemon answers
+        #   `cannot set both Count and DeviceIDs on device request` (nvidia-docker#1026).
+        #   The documented form is --gpus '"device=0,1"' under a shell; this goes straight
+        #   to argv with no shell, so the double quotes have to stay inside the string. A
+        #   single card is quoted too: same parse, one fewer branch.
+        return ["--gpus", f'"device={ids}"']
+
+    # ── The six actions ─────────────────────────────────────────────────────
+
+    async def run(self, spec: ContainerSpec) -> str:
+        args = ["run", "-d", "--name", spec.name]
+
+        # ★ No --rm. The exit code and the logs are still needed once the container has
+        #   exited; reclaiming it is reap()'s job, with the same semantics as KubeRay's
+        #   ttlSecondsAfterFinished.
+
+        args += self._gpu_args(spec.gpus)
+        for k, v in spec.env.items():
+            args += ["-e", f"{k}={v}"]
+        for k, v in spec.labels.items():
+            args += ["--label", f"{k}={v}"]
+        for host, cont, ro in spec.mounts:
+            args += ["-v", f"{host}:{cont}:ro" if ro else f"{host}:{cont}"]
+        if spec.workdir:
+            args += ["-w", spec.workdir]
+        if spec.shm_size:
+            args += ["--shm-size", spec.shm_size]
+        if spec.cpu_limit:
+            args += ["--cpus", spec.cpu_limit]
+        if spec.memory_limit:
+            args += ["--memory", spec.memory_limit]
+        if spec.network:
+            args += ["--network", spec.network]
+        if spec.user:
+            args += ["--user", spec.user]
+        # A training process has to be able to finish cleanly on stop (to write a
+        # checkpoint), so leave the signal a path to reach it.
+        args += ["--stop-signal", "SIGTERM"]
+        # `command` is the entry argv (see ContainerSpec), so it has to *replace*
+        # the image's ENTRYPOINT rather than be appended to it as CMD.
+        #
+        # Without `--entrypoint` it is appended, and an image whose entrypoint is
+        # a CLI rather than an exec-wrapper receives our argv as its own flags. A
+        # Playground session on `vllm/vllm-openai` -- whose ENTRYPOINT is the
+        # `vllm` command -- died at once with
+        #     vllm: error: unrecognized arguments: -lc python3 -m vllm...
+        # which is this `bash -lc` arriving where a subcommand was expected. The
+        # same launch works on kuberay, because a Kubernetes container's
+        # `command` overrides the entrypoint by definition -- so the two backends
+        # disagreed about the one field that says what to run, and the process
+        # runtime (which execs the argv directly) agreed with Kubernetes.
+        #
+        # What this gives up is an image's own entrypoint wrapper, such as
+        # nvidia_entrypoint.sh on the NGC bases. That wrapper prints a banner and
+        # `exec "$@"`, and the kuberay path has been skipping it since it was
+        # written, which is the evidence that nothing depends on it.
+        if spec.command:
+            args += ["--entrypoint", spec.command[0]]
+        args.append(spec.image)
+        args += spec.command[1:]
+
+        _, out, _ = await self._exec(*args)
+        return out.strip()
+
+    async def inspect(self, name: str) -> ContainerState:
+        code, out, _ = await self._exec("inspect", name, check=False)
+        if code != 0:
+            return ContainerState(name=name, exists=False)
+        try:
+            data = json.loads(out)[0]
+        except (json.JSONDecodeError, IndexError, KeyError):
+            return ContainerState(name=name, exists=False)
+        return _state_from_inspect(name, data)
+
+    async def logs(self, name: str, *, tail_lines: int = 2000) -> str:
+        # docker logs keeps the training run's stdout and stderr apart; both are wanted.
+        code, out, err = await self._exec(
+            "logs", "--timestamps", "--tail", str(tail_lines), name, check=False
+        )
+        if code != 0:
+            return f"(the container logs could not be read: {err.strip()[:200]})"
+        return out + err
+
+    async def stop(self, name: str, *, timeout: int = 30) -> bool:
+        code, _, err = await self._exec("stop", "-t", str(timeout), name, check=False)
+        if code == 0:
+            return True
+        # Already gone = the goal is met, not a failure.
+        if "no such container" in err.lower():
+            return True
+        log.warning("failed to stop container %s: %s", name, err.strip()[:200])
+        return False
+
+    async def remove(self, name: str) -> bool:
+        code, _, err = await self._exec("rm", "-f", name, check=False)
+        return code == 0 or "no such container" in err.lower()
+
+    async def ps(self) -> list[ContainerState]:
+        """List every container this platform owns, exited ones included.
+
+        `--filter label=` filters on the key alone, so what comes back is the platform's
+        own containers and nothing else running on the host is touched -- which matters,
+        because reclaim relies on it.
+        """
+        _, out, _ = await self._exec(
+            "ps", "-a", "--filter", f"label={LABEL_RUN_ID}", "--format", "{{.Names}}"
+        )
+        names = [n.strip() for n in out.splitlines() if n.strip()]
+        if not names:
+            return []
+        states = await asyncio.gather(
+            *(self.inspect(n) for n in names), return_exceptions=True
+        )
+        return [s for s in states if isinstance(s, ContainerState) and s.exists]
+
+    async def health(self) -> dict:
+        try:
+            _, out, _ = await self._exec("version", "--format", "{{.Server.Version}}")
+        except RuntimeError_ as e:
+            return {"backend": self.name, "ok": False, "detail": str(e)}
+        return {"backend": self.name, "ok": True, "version": out.strip()}
+
+    async def image_exists(self, image: str) -> bool:
+        code, _, _ = await self._exec("image", "inspect", image, check=False)
+        return code == 0
+
+    async def pull(self, image: str, *, timeout: float) -> None:
+        """Pull an image explicitly, on a timeout of its own.
+
+        A training image runs to tens of GB and does not fit inside the default CLI
+        timeout. Skip this and `run` pulls implicitly and hits local_cli_timeout (60s):
+        the first job to use a new image always fails to start, reported as nothing but
+        a timeout, which is close to undiagnosable from the outside.
+        """
+        await self._exec("pull", image, timeout=timeout)
+
+
+def _state_from_inspect(name: str, data: dict) -> ContainerState:
+    state = data.get("State") or {}
+    cfg = data.get("Config") or {}
+    status = str(state.get("Status") or "").lower()
+    # ExitCode is 0 while running, which means "has not exited yet" rather than "exited
+    # successfully" -- taking it as 0 records a running job as SUCCEEDED in the ledger,
+    # releasing its quota while the cards are still held.
+    exit_code = None if status in ("running", "created", "paused") else state.get("ExitCode")
+    return ContainerState(
+        name=name,
+        exists=True,
+        status=status,
+        exit_code=exit_code if exit_code is None else int(exit_code),
+        labels={k: str(v) for k, v in (cfg.get("Labels") or {}).items()},
+        created_at=str(data.get("Created") or ""),
+        started_at=str(state.get("StartedAt") or ""),
+        finished_at=str(state.get("FinishedAt") or ""),
+        oom_killed=bool(state.get("OOMKilled")),
+    )
+
+
+# ── Bare process runtime ─────────────────────────────────────────────────────
+
+
+class ProcessRuntime:
+    """The fallback when there is no container runtime. **No isolation; dev machines only.**
+
+    Three real differences from the container modes, to be known before choosing it:
+
+      1. Held GPU memory comes back. A container exiting destroys its cgroup and the
+         memory is certain to be released; a bare process that crashes can leave a
+         zombie holding a card, which is the reason containers were adopted here.
+      2. No resource boundary. One job going OOM takes the whole machine with it.
+      3. State has to be stored here. docker keeps a container's metadata; a bare
+         process is only a pid -- after a console restart the pid alone cannot give the
+         exit code, so this writes a state file.
+    """
+
+    name = "process"
+
+    def __init__(self, state_dir: Path, *, timeout: float = 60.0):
+        self.state_dir = Path(state_dir)
+        self.timeout = timeout
+
+    def _meta(self, name: str) -> Path:
+        return self.state_dir / f"{name}.json"
+
+    def _log(self, name: str) -> Path:
+        return self.state_dir / f"{name}.log"
+
+    def _rc(self, name: str) -> Path:
+        return self.state_dir / f"{name}.rc"
+
+    async def run(self, spec: ContainerSpec) -> str:
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        env = {**os.environ, **spec.env}
+        if spec.gpus:
+            env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in spec.gpus)
+        else:
+            # Set empty rather than left unset: unset, the process sees every card and
+            # can fill them all.
+            env["CUDA_VISIBLE_DEVICES"] = ""
+
+        rc_path, log_path = self._rc(spec.name), self._log(spec.name)
+        rc_path.unlink(missing_ok=True)
+
+        # The exit code has to stay readable across a console restart, so wrap the
+        # command in a shell that writes it to a file. That shell does nothing but run
+        # the command and record its status; the user's command arrives through "$@"
+        # rather than being interpolated -- interpolation is injection.
+        wrapper = 'set -o pipefail; "$@"; echo $? > "$TUNEPLANE_RC_FILE"'
+        env["TUNEPLANE_RC_FILE"] = str(rc_path)
+
+        with open(log_path, "wb") as fh:
+            proc = await asyncio.create_subprocess_exec(
+                "bash", "-c", wrapper, "bash", *spec.command,
+                stdout=fh, stderr=asyncio.subprocess.STDOUT,
+                cwd=spec.workdir or None,
+                env=env,
+                start_new_session=True,   # its own process group, so stop can kill all of it
+            )
+        self._meta(spec.name).write_text(json.dumps({
+            "name": spec.name, "pid": proc.pid, "labels": spec.labels,
+            "started_at": time.time(),
+        }), encoding="utf-8")
+        return str(proc.pid)
+
+    def _alive(self, pid: int) -> bool:
+        try:
+            os.kill(pid, 0)
+        except ProcessLookupError:
+            return False
+        except PermissionError:
+            return True     # it exists, it just belongs to another user
+        return True
+
+    async def inspect(self, name: str) -> ContainerState:
+        meta_path = self._meta(name)
+        if not meta_path.is_file():
+            return ContainerState(name=name, exists=False)
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            return ContainerState(name=name, exists=False)
+
+        rc_path = self._rc(name)
+        if rc_path.is_file():
+            try:
+                code = int(rc_path.read_text(encoding="utf-8").strip() or 1)
+            except ValueError:
+                code = 1
+            return ContainerState(
+                name=name, exists=True, status="exited", exit_code=code,
+                labels=meta.get("labels") or {},
+            )
+        if self._alive(int(meta.get("pid") or 0)):
+            return ContainerState(
+                name=name, exists=True, status="running", labels=meta.get("labels") or {}
+            )
+        # The process is gone but wrote no exit code: it was SIGKILLed, and the OOM
+        # killer is the usual reason.
+        return ContainerState(
+            name=name, exists=True, status="exited", exit_code=137,
+            labels=meta.get("labels") or {},
+        )
+
+    async def logs(self, name: str, *, tail_lines: int = 2000) -> str:
+        path = self._log(name)
+        if not path.is_file():
+            return ""
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+        return "\n".join(lines[-tail_lines:])
+
+    async def stop(self, name: str, *, timeout: int = 30) -> bool:
+        meta_path = self._meta(name)
+        if not meta_path.is_file():
+            return True
+        try:
+            pid = int(json.loads(meta_path.read_text(encoding="utf-8")).get("pid") or 0)
+        except (json.JSONDecodeError, ValueError):
+            return True
+        if not pid:
+            return True
+        # Kill the whole process group: training spawns a crowd of children (Ray
+        # workers, dataloaders), and killing only the parent leaves orphans behind
+        # still holding GPU memory.
+        for sig, wait in ((signal.SIGTERM, timeout), (signal.SIGKILL, 0)):
+            try:
+                os.killpg(os.getpgid(pid), sig)
+            except (ProcessLookupError, PermissionError):
+                return True
+            if not wait:
+                break
+            for _ in range(wait * 2):
+                await asyncio.sleep(0.5)
+                if not self._alive(pid):
+                    return True
+        return True
+
+    async def remove(self, name: str) -> bool:
+        for p in (self._meta(name), self._log(name), self._rc(name)):
+            p.unlink(missing_ok=True)
+        return True
+
+    async def ps(self) -> list[ContainerState]:
+        if not self.state_dir.is_dir():
+            return []
+        out = []
+        for meta in sorted(self.state_dir.glob("*.json")):
+            state = await self.inspect(meta.stem)
+            if state.exists:
+                out.append(state)
+        return out
+
+    async def health(self) -> dict:
+        return {
+            "backend": self.name, "ok": True,
+            "detail": "bare-process mode: no isolation, and GPU memory may be left behind -- for a development machine only",
+        }
+
+    async def image_exists(self, image: str) -> bool:
+        del image
+        return True     # a bare process has no image, so it is always ready
+
+    async def pull(self, image: str, *, timeout: float) -> None:
+        del image, timeout
+
+
+# ── Detection ────────────────────────────────────────────────────────────────
+
+
+def detect_runtime(preference: str, *, state_dir: Path, timeout: float = 60.0) -> ContainerRuntime:
+    """Build the runtime the preference names.
+
+    `auto` tries docker then podman, and **raises when neither is there rather than
+    falling back to process**.
+
+    A silent downgrade has two consequences, and the second one is fatal:
+      1. It leaves people believing they run containerised jobs, until held GPU memory
+         one day shows there was no isolation.
+      2. ★ process mode keeps its state in a state_dir of its own. Jobs started under
+         docker are invisible when looked up in process mode -- reconciliation then
+         judges **every running job finished**, releasing quota while the cards stay
+         held. One misconfiguration is enough to scramble the whole ledger.
+
+    To run bare processes, set TUNEPLANE_LOCAL_RUNTIME=process explicitly, so that the
+    choice is on the record in the configuration.
+    """
+    pref = (preference or "auto").strip().lower()
+
+    if pref == "process":
+        return ProcessRuntime(state_dir, timeout=timeout)
+
+    if pref in ("docker", "podman"):
+        path = shutil.which(pref)
+        if not path:
+            raise RuntimeError_(
+                f"TUNEPLANE_LOCAL_RUNTIME={pref} but {pref} is not on PATH. "
+                f"Use auto to fall back automatically, or process (no isolation)."
+            )
+        return CliRuntime(path, timeout=timeout)
+
+    if pref != "auto":
+        raise RuntimeError_(
+            f"unknown runtime {preference!r} (one of: auto, docker, podman, process)"
+        )
+
+    for candidate in ("docker", "podman"):
+        if path := shutil.which(candidate):
+            return CliRuntime(path, timeout=timeout)
+    raise RuntimeError_(
+        "Neither docker nor podman is on PATH. The local backend needs a container runtime; "
+        "to really run bare processes (no isolation, GPU memory may be left held), set "
+        "TUNEPLANE_LOCAL_RUNTIME=process explicitly."
+    )
+
+
+def age_seconds(finished_at: str, now: float) -> float:
+    """How long ago a container exited. Unparseable means "long ago".
+
+    Reclaiming an exited container early costs nothing; letting wreckage pile up
+    because a timestamp format did not parse costs disk on every node.
+    """
+    if not finished_at:
+        return float("inf")
+    from datetime import datetime
+
+    try:
+        ts = datetime.fromisoformat(finished_at.replace("Z", "+00:00"))
+    except ValueError:
+        return float("inf")
+    return max(0.0, now - ts.timestamp())

tuneplane_node/settings.py

@@ -0,0 +1,77 @@
+"""What a node needs to know about itself.
+
+A deliberate subset of the console's `WebSettings`, sharing its `TUNEPLANE_` prefix
+so an operator configures both the same way and a machine that is also the
+console reads one file.
+
+**Small on purpose.** The console has 235 settings and a node needs thirteen of
+them; importing the big one to get those thirteen is what dragged
+`python-jose`, `passlib` and SQLAlchemy onto every GPU box. Anything added here
+should be something the machine itself has to answer -- where its containers run,
+which cards it has, where the shared filesystem is. Everything about scheduling,
+quota, auth and storage policy belongs to the console.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from tuneplane.layout import StorageLayout
+
+
+class NodeSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="TUNEPLANE_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        env_ignore_empty=True,
+        extra="ignore",
+    )
+
+    # ── who this node answers to ────────────────────────────────────────────
+    #: Shared with the console. Missing refuses startup rather than exposing the
+    #: node's container API unauthenticated.
+    node_token: str = ""
+    #: Where the identity and credential a join produced are kept. Not under the
+    #: storage root: a node that cannot see the shared mount still has to be able
+    #: to say who it is.
+    node_state_dir: str = "~/.tuneplane-node"
+
+    # ── how it runs containers ──────────────────────────────────────────────
+    local_runtime: str = ""
+    local_cli_timeout: float = 60.0
+    local_stop_timeout: int = 30
+    local_container_ttl_s: int = 86400
+    local_image_pull_timeout_s: float = 3600.0
+    local_gpu_passthrough: str = ""
+
+    # ── what cards it has ───────────────────────────────────────────────────
+    local_gpu_count: int = 0
+    local_check_gpu_health: bool = True
+    local_check_external_gpus: bool = True
+    #: The profile an operator declared for this machine. Reported to the console
+    #: as-is: mapping a profile or a card name to a hardware series is the
+    #: console's registry to own, and a node that guessed would be a second
+    #: answer to a question one place already owns.
+    cluster_profile: str = ""
+
+    # ── where the shared filesystem is ──────────────────────────────────────
+    storage_root: Optional[Path] = None
+
+    @property
+    def storage_configured(self) -> bool:
+        return self.storage_root is not None
+
+    @property
+    def storage(self) -> StorageLayout:
+        """The same layout the console derives, from the same root.
+
+        Falls back to `./.tuneplane-data/storage` so a developer running the agent in a
+        checkout gets a working tree without configuring anything -- the same
+        rule `WebSettings.storage` follows, and it has to be the same rule or the
+        two sides disagree about where a job's files are.
+        """
+        root = self.storage_root or Path(os.getcwd()) / ".tuneplane-data" / "storage"
+        return StorageLayout(root)