fadenstack 0.4.2__py3-none-any.whl

faden_cli/core/runtime_images.py

@@ -0,0 +1,309 @@
+"""Runtime images: the containers models run in, fetched onto the server.
+
+A machine never pulls its runtime image from the internet. The Fadenstack
+server exports the pinned image from its own Docker and the machine downloads
+it from the server, or from a cluster peer that already has it. So the server
+has to hold every runtime image a machine may ask for -- and nothing put it
+there: a fresh install could take over a cluster that already had its image,
+and could not give a new machine one.
+
+``deploy`` and ``upgrade`` now fetch them from the registry the release
+publishes to (``ghcr.io/fadenstack/ray-runtime-*``). Which images is not decided
+here: they are read out of this release's backend image, from the same
+runtime manifests the backend builds its catalogue from, so the CLI can never
+fetch a build the backend would then refuse.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from faden_cli.core.env_gen import read_env_file
+
+#: Architectures as machines report them (``uname -m``), with the names
+#: registries and Docker use for the same thing.
+ARCHITECTURES = ("x86_64", "aarch64")
+_ARCH_ALIASES = {"x86_64": "x86_64", "amd64": "x86_64", "aarch64": "aarch64", "arm64": "aarch64"}
+_PLATFORMS = {"x86_64": "linux/amd64", "aarch64": "linux/arm64"}
+
+#: Run inside the backend image, whose working directory holds the manifests.
+_READ_MANIFESTS = (
+    "import glob, json\n"
+    "print(json.dumps({p.rsplit('/', 1)[-1]: json.load(open(p))"
+    " for p in sorted(glob.glob('runtime-image/runtime-manifest*.json'))}))\n"
+)
+
+DEFAULT_REGISTRY = "ghcr.io/fadenstack"
+
+
+@dataclass(frozen=True)
+class RuntimeImage:
+    """One runtime image a release pins."""
+
+    manifest: str
+    ref: str
+    arch: str
+    rootfs_layers: tuple[str, ...]
+
+    @property
+    def platform(self) -> str | None:
+        """Docker's name for the architecture (``linux/arm64``), or None when unknown."""
+        return _PLATFORMS.get(self.arch)
+
+    @property
+    def published(self) -> bool:
+        """Whether the reference names a registry to pull it from.
+
+        Until a runtime release is certified, a manifest can still name a
+        build that only ever existed on the machines it was made on
+        (``faden/ray-vllm-gb10:…``); pulling that would ask Docker Hub for
+        an image nobody published.
+        """
+        return is_registry_ref(self.ref)
+
+
+@dataclass
+class Outcome:
+    """What became of one image."""
+
+    image: RuntimeImage
+    status: str  # present | pulled | skipped | not_published | failed | mismatch
+    detail: str = ""
+
+
+@dataclass
+class Report:
+    """Everything ``fetch`` did, for the caller to print."""
+
+    backend_image: str
+    outcomes: list[Outcome] = field(default_factory=list)
+    error: str = ""
+
+    @property
+    def ok(self) -> bool:
+        """Whether everything that was wanted is on the server."""
+        return not self.error and not any(o.status in {"failed", "mismatch"} for o in self.outcomes)
+
+
+def is_registry_ref(ref: str) -> bool:
+    """``ghcr.io/org/name:tag`` yes; ``org/name:tag`` or ``name:tag`` no (they mean Docker Hub)."""
+    first, sep, _ = ref.partition("/")
+    return bool(sep) and ("." in first or ":" in first or first == "localhost")
+
+
+def normalise_arch(value: str) -> str:
+    """``amd64`` -> ``x86_64``, ``arm64`` -> ``aarch64``; empty for anything else."""
+    return _ARCH_ALIASES.get((value or "").strip().lower(), "")
+
+
+def parse_selection(value: str | None) -> set[str]:
+    """``all`` / ``none`` / ``x86_64,aarch64`` -> the architectures to fetch.
+
+    Raises:
+        ValueError: an architecture nobody builds a runtime for.
+    """
+    text = (value or "all").strip().lower()
+    if text in {"", "all"}:
+        return set(ARCHITECTURES)
+    if text == "none":
+        return set()
+    chosen: set[str] = set()
+    for part in text.split(","):
+        arch = normalise_arch(part)
+        if not arch:
+            raise ValueError(f"unknown architecture {part.strip()!r}; use all, none, x86_64 or aarch64")
+        chosen.add(arch)
+    return chosen
+
+
+def backend_image(env_path: Path) -> str:
+    """The backend image this install runs, as the compose file names it."""
+    env = read_env_file(env_path) if env_path.exists() else {}
+    registry = (env.get("REGISTRY") or DEFAULT_REGISTRY).rstrip("/")
+    version = env.get("VERSION") or "latest"
+    return f"{registry}/backend:{version}"
+
+
+def _arch_of(manifest: dict) -> str:
+    for value in (
+        manifest.get("machine"),
+        (manifest.get("target_hardware") or {}).get("architecture"),
+        (manifest.get("base_image") or {}).get("architecture"),
+    ):
+        arch = normalise_arch(str(value or ""))
+        if arch:
+            return arch
+    return ""
+
+
+def _docker() -> str:
+    docker = shutil.which("docker")
+    if not docker:
+        raise RuntimeError("docker is not on PATH")
+    return docker
+
+
+def _run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(cmd, check=False, text=True, **kwargs)  # noqa: S603
+
+
+def pinned(image: str) -> list[RuntimeImage]:
+    """The runtime images *image* (a backend image) pins, read out of it.
+
+    Raises:
+        RuntimeError: the backend image cannot be run or holds no manifests.
+    """
+    proc = _run(
+        [_docker(), "run", "--rm", "--entrypoint", "python", image, "-c", _READ_MANIFESTS],
+        capture_output=True,
+    )
+    if proc.returncode != 0:
+        detail = (proc.stderr or proc.stdout or "").strip().splitlines()[-1:] or [""]
+        raise RuntimeError(f"could not read the runtime manifests from {image}: {detail[0]}")
+    manifests: dict[str, dict] = {}
+    for line in reversed((proc.stdout or "").splitlines()):
+        if line.strip().startswith("{"):
+            manifests = json.loads(line)
+            break
+    images = []
+    for name, manifest in manifests.items():
+        ref = str(manifest.get("release_tag") or manifest.get("target_image") or "")
+        if not ref:
+            continue
+        images.append(
+            RuntimeImage(
+                manifest=name,
+                ref=ref,
+                arch=_arch_of(manifest),
+                rootfs_layers=tuple(manifest.get("rootfs_layers") or ()),
+            )
+        )
+    return images
+
+
+def local_layers(ref: str) -> tuple[str, ...] | None:
+    """The layers of *ref* in this server's Docker, or None when it is not here."""
+    proc = _run(
+        [_docker(), "image", "inspect", "--format", "{{json .RootFS.Layers}}", ref],
+        capture_output=True,
+    )
+    if proc.returncode != 0:
+        return None
+    try:
+        return tuple(json.loads(proc.stdout.strip() or "[]"))
+    except ValueError:
+        return None
+
+
+def pull(image: RuntimeImage) -> int:
+    """``docker pull`` it, for its own platform, showing Docker's progress."""
+    cmd = [_docker(), "pull"]
+    if image.platform:
+        # The GB10 image exists only for arm64. An x86_64 server that hands it
+        # to DGX machines has to be told which platform it wants, or Docker
+        # finds "no matching manifest" for its own.
+        cmd += ["--platform", image.platform]
+    cmd.append(image.ref)
+    return _run(cmd).returncode
+
+
+def fetch(image: str, *, architectures: set[str]) -> Report:
+    """Put every runtime image *image* pins, for the chosen architectures, on this server."""
+    report = Report(backend_image=image)
+    try:
+        images = pinned(image)
+    except RuntimeError as exc:
+        report.error = str(exc)
+        return report
+    if not images:
+        report.error = f"{image} names no runtime images"
+        return report
+
+    for runtime in images:
+        if runtime.arch and runtime.arch not in architectures:
+            report.outcomes.append(Outcome(runtime, "skipped", f"{runtime.arch} not chosen"))
+            continue
+        # Identity is the content, as the backend checks it: the layer list.
+        have = local_layers(runtime.ref)
+        if have is not None and have == runtime.rootfs_layers:
+            report.outcomes.append(Outcome(runtime, "present"))
+            continue
+        if not runtime.published:
+            report.outcomes.append(
+                Outcome(runtime, "not_published", "names no registry; load it with docker load"),
+            )
+            continue
+        if pull(runtime) != 0:
+            report.outcomes.append(Outcome(runtime, "failed", "docker pull failed"))
+            continue
+        have = local_layers(runtime.ref)
+        if runtime.rootfs_layers and have != runtime.rootfs_layers:
+            # The tag was moved, or the registry serves another build: the
+            # backend would refuse to hand this to a machine.
+            report.outcomes.append(
+                Outcome(runtime, "mismatch", "the registry's image is not the build this release pins"),
+            )
+            continue
+        report.outcomes.append(Outcome(runtime, "pulled"))
+    return report
+
+
+# ── the machines' model-TLS proxy ──────────────────────────────────────────
+
+#: nginx in front of the models on every machine (the node agent's
+#: ``model_tls.py`` pins the same build): one single-platform manifest per
+#: architecture, and the content identity the agent checks (sha256 over the
+#: ordered layer diff IDs). An air-gapped machine takes it from this server.
+MODEL_TLS_NGINX = {
+    "x86_64": (
+        "nginx@sha256:0530961ff0592b58c10f767535cc0abdfccf9e389ff7cc90f87320c1bc7e8506",
+        "sha256:268c8b4b4baa285e4082ade32c9f2f69a013617ea0b7d0ff25f711bdb10750e7",
+    ),
+    "aarch64": (
+        "nginx@sha256:84dd96a0337ae0f15be81afa8c6bd99d06a514a2ab71a1503aba91ce4392ecc2",
+        "sha256:2f89fd28b672e74bd0d11deb485485881ca606ccc04a6a14b05cca1272f0459a",
+    ),
+}
+MODEL_TLS_TAG = "faden/model-tls-nginx:1.31.6-alpine-{arch}"
+
+
+def rootfs_digest(layers: tuple[str, ...] | list[str]) -> str:
+    """What the agent and the backend compare: sha256 over the layer IDs, one per line."""
+    import hashlib  # noqa: PLC0415
+
+    return "sha256:" + hashlib.sha256(chr(10).join(layers).encode()).hexdigest()
+
+
+def fetch_model_tls(architectures: set[str]) -> list[tuple[str, str, str]]:
+    """Hold the model-TLS proxy image for each chosen architecture: ``(tag, arch, outcome)``.
+
+    Pulled by its single-platform manifest (so either Docker image store keeps
+    exactly that platform under the tag) and checked against the pin before
+    it is tagged where the agents ask for it.
+    """
+    out: list[tuple[str, str, str]] = []
+    for arch, (ref, pin) in MODEL_TLS_NGINX.items():
+        tag = MODEL_TLS_TAG.format(arch=arch)
+        if arch not in architectures:
+            out.append((tag, arch, "skipped"))
+            continue
+        have = local_layers(tag)
+        if have and rootfs_digest(have) == pin:
+            out.append((tag, arch, "present"))
+            continue
+        if _run([_docker(), "pull", "--platform", _PLATFORMS[arch], ref]).returncode != 0:
+            out.append((tag, arch, "failed"))
+            continue
+        pulled = local_layers(ref)
+        if not pulled or rootfs_digest(pulled) != pin:
+            out.append((tag, arch, "mismatch"))
+            continue
+        if _run([_docker(), "tag", ref, tag], capture_output=True).returncode != 0:
+            out.append((tag, arch, "failed"))
+            continue
+        out.append((tag, arch, "pulled"))
+    return out

faden_cli/core/schedule.py

@@ -0,0 +1,278 @@
+"""Daily jobs -- the backup, the certificate renewal -- for the user who runs Fadenstack; no root.
+
+Two ways, whichever the machine has:
+
+* a **systemd user timer** (``faden-backup.timer``), preferred: every
+  systemd distribution has one, and ``Persistent=true`` runs a job missed
+  while the server was off. It runs without anyone logged in only with
+  *lingering* on, which is switched on for the user (as the node agent's
+  installer does).
+* a **crontab line** marked with the job's name, where there is cron but no
+  usable systemd user manager.
+
+Ubuntu 26.04 server ships no cron, which is why cron is not the only way.
+"""
+
+from __future__ import annotations
+
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+#: Marks the crontab line of the backup.
+MARK = "# faden-backup"
+#: The systemd user units of the backup.
+UNIT = "faden-backup"
+#: The systemd user units (and crontab mark) of the daily ``faden tls renew``.
+TLS_UNIT = "faden-tls-renew"
+
+#: Scheduled jobs run with a bare PATH; docker is usually in one of these.
+_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+
+def _run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(cmd, capture_output=True, text=True, check=False, **kwargs)  # noqa: S603
+
+
+# ── which way ─────────────────────────────────────────────────────
+
+
+def _systemd_user() -> bool:
+    if sys.platform != "linux" or shutil.which("systemctl") is None:
+        return False
+    state = _run(["systemctl", "--user", "is-system-running"]).stdout.strip()
+    # "degraded" still runs timers; "offline"/empty means no user manager.
+    return state in {"running", "degraded", "starting", "initializing"}
+
+
+def _cron() -> bool:
+    return sys.platform != "win32" and shutil.which("crontab") is not None
+
+
+def available() -> bool:
+    """Whether jobs can be scheduled on this machine at all."""
+    return _systemd_user() or _cron()
+
+
+# ── what runs ─────────────────────────────────────────────────────
+
+
+def _cli() -> list[str]:
+    """How a scheduled job should run this CLI: by absolute path, not a PATH lookup."""
+    found = shutil.which("faden")
+    if found:
+        return [str(Path(found).resolve())]
+    return [sys.executable, "-m", "faden_cli"]
+
+
+def _config() -> Path | None:
+    value = os.environ.get("FADEN_CONFIG")
+    return Path(value).expanduser().resolve() if value else None
+
+
+@dataclass(frozen=True)
+class Job:
+    """A command of this CLI run every day at *hour*:*minute*, appending to *log*."""
+
+    unit: str
+    description: str
+    args: tuple[str, ...]
+    hour: int
+    minute: int
+    log: Path
+
+    @property
+    def mark(self) -> str:
+        """What ends its crontab line."""
+        return f"# {self.unit}"
+
+
+def _command(job: Job) -> str:
+    return " ".join(shlex.quote(part) for part in [*_cli(), *job.args])
+
+
+# ── cron ──────────────────────────────────────────────────────────
+
+
+def _read() -> list[str]:
+    out = _run(["crontab", "-l"])
+    # "no crontab for <user>" is exit 1 with nothing to keep.
+    return out.stdout.splitlines() if out.returncode == 0 else []
+
+
+def _write(lines: list[str]) -> None:
+    text = "\n".join(lines).rstrip("\n") + "\n" if lines else ""
+    subprocess.run(["crontab", "-"], input=text, text=True, check=True)  # noqa: S607
+
+
+def job_entry(job: Job, config: Path | None) -> str:
+    """The crontab line for *job*."""
+    env = f"PATH={_PATH}"
+    if config is not None:
+        env += f" FADEN_CONFIG={shlex.quote(str(config))}"
+    log = shlex.quote(str(job.log))
+    return f"{job.minute} {job.hour} * * * {env} {_command(job)} >> {log} 2>&1 {job.mark}"
+
+
+# ── systemd ───────────────────────────────────────────────────────
+
+
+def _unit_dir() -> Path:
+    base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
+    return Path(base) / "systemd" / "user"
+
+
+def job_units(job: Job, config: Path | None) -> dict[str, str]:
+    """The service and timer unit files for *job*."""
+    env = [f"Environment=PATH={_PATH}"]
+    if config is not None:
+        env.append(f'Environment="FADEN_CONFIG={config}"')
+    service = "\n".join([
+        "[Unit]",
+        f"Description={job.description}",
+        "",
+        "[Service]",
+        "Type=oneshot",
+        *env,
+        f"ExecStart={_command(job)}",
+        f"StandardOutput=append:{job.log}",
+        f"StandardError=append:{job.log}",
+        "",
+    ])
+    timer = "\n".join([
+        "[Unit]",
+        f"Description={job.description}",
+        "",
+        "[Timer]",
+        f"OnCalendar=*-*-* {job.hour:02d}:{job.minute:02d}:00",
+        # A day the server was off is caught up when it next starts.
+        "Persistent=true",
+        "",
+        "[Install]",
+        "WantedBy=timers.target",
+        "",
+    ])
+    return {f"{job.unit}.service": service, f"{job.unit}.timer": timer}
+
+
+def _linger() -> bool:
+    """Lingering on for this user (so the timer runs with nobody logged in); whether it is."""
+    user = os.environ.get("USER") or ""
+    shown = _run(["loginctl", "show-user", user, "-p", "Linger"]).stdout.strip()
+    if shown == "Linger=yes":
+        return True
+    _run(["loginctl", "enable-linger", user])
+    return _run(["loginctl", "show-user", user, "-p", "Linger"]).stdout.strip() == "Linger=yes"
+
+
+# ── any job ───────────────────────────────────────────────────────
+
+
+def job_current(unit: str) -> str | None:
+    """How *unit* is scheduled, if it is: the timer's time or the crontab line."""
+    enabled = ""
+    if _systemd_user():
+        enabled = _run(["systemctl", "--user", "is-enabled", f"{unit}.timer"]).stdout.strip()
+    if enabled == "enabled":
+        lines = (_unit_dir() / f"{unit}.timer").read_text().splitlines()
+        calendar = next((line for line in lines if line.startswith("OnCalendar=")), "")
+        return f"systemd timer {unit}.timer ({calendar.removeprefix('OnCalendar=')})"
+    if _cron():
+        return next((line for line in _read() if line.endswith(f"# {unit}")), None)
+    return None
+
+
+def install_job(job: Job) -> tuple[str, list[str]]:
+    """Schedule *job*, replacing an earlier schedule of it. Returns how, and warnings for the operator."""
+    config = _config()
+    warnings: list[str] = []
+    remove_job(job.unit)
+    job.log.parent.mkdir(parents=True, exist_ok=True)
+    if _systemd_user():
+        directory = _unit_dir()
+        directory.mkdir(parents=True, exist_ok=True)
+        for name, text in job_units(job, config).items():
+            (directory / name).write_text(text, encoding="utf-8")
+        _run(["systemctl", "--user", "daemon-reload"])
+        enabled = _run(["systemctl", "--user", "enable", "--now", f"{job.unit}.timer"])
+        if enabled.returncode != 0:
+            msg = f"systemctl --user enable {job.unit}.timer failed: {enabled.stderr.strip()}"
+            raise RuntimeError(msg)
+        if not _linger():
+            warnings.append(
+                f"Lingering is off for this user, so {job.unit} only runs while someone is logged in. "
+                f"Turn it on once with: sudo loginctl enable-linger {os.environ.get('USER', '<user>')}"
+            )
+        return f"systemd timer {job.unit}.timer", warnings
+    if _cron():
+        _write([*_read(), job_entry(job, config)])
+        return "crontab", warnings
+    msg = "Neither a systemd user manager nor cron is available."
+    raise RuntimeError(msg)
+
+
+def remove_job(unit: str) -> bool:
+    """Remove *unit*'s schedule, whichever way it was scheduled; whether there was one."""
+    removed = False
+    if _systemd_user():
+        timer = _unit_dir() / f"{unit}.timer"
+        if timer.exists():
+            _run(["systemctl", "--user", "disable", "--now", f"{unit}.timer"])
+            for name in (f"{unit}.timer", f"{unit}.service"):
+                (_unit_dir() / name).unlink(missing_ok=True)
+            _run(["systemctl", "--user", "daemon-reload"])
+            removed = True
+    if _cron():
+        lines = _read()
+        kept = [line for line in lines if not line.endswith(f"# {unit}")]
+        if len(kept) != len(lines):
+            _write(kept)
+            removed = True
+    return removed
+
+
+# ── the backup ────────────────────────────────────────────────────
+
+
+def _backup(*, hour: int, minute: int, retain: int, log: Path) -> Job:
+    return Job(UNIT, "Fadenstack nightly backup", ("backup", "-y", "--retain", str(retain)), hour, minute, log)
+
+
+def entry(*, hour: int, minute: int, retain: int, log: Path, config: Path | None) -> str:
+    """The crontab line for a backup every day at *hour*:*minute*."""
+    return job_entry(_backup(hour=hour, minute=minute, retain=retain, log=log), config)
+
+
+def units(*, hour: int, minute: int, retain: int, log: Path, config: Path | None) -> dict[str, str]:
+    """The service and timer unit files for a backup every day at *hour*:*minute*."""
+    return job_units(_backup(hour=hour, minute=minute, retain=retain, log=log), config)
+
+
+def current() -> str | None:
+    """How the nightly backup is scheduled, if it is: the timer's time or the crontab line."""
+    return job_current(UNIT)
+
+
+def install(*, hour: int, minute: int, retain: int, log: Path) -> tuple[str, list[str]]:
+    """Schedule the backup, replacing an earlier schedule.
+
+    Returns how it is scheduled, and warnings for the operator.
+    """
+    return install_job(_backup(hour=hour, minute=minute, retain=retain, log=log))
+
+
+def remove() -> bool:
+    """Remove the scheduled backup, whichever way it was scheduled; whether there was one."""
+    return remove_job(UNIT)
+
+
+# ── the certificate renewal ───────────────────────────────────────
+
+
+def tls_renewal(log: Path) -> Job:
+    """``faden tls renew`` every morning; it changes nothing until something is due."""
+    return Job(TLS_UNIT, "Fadenstack certificate renewal", ("tls", "renew"), 4, 40, log)