flashruntime 0.3.0__py3-none-any.whl

flashruntime/viewer/server.py

@@ -0,0 +1,134 @@
+"""RunViewerServer — the read-only HTTP surface over one run directory.
+
+A stdlib `ThreadingHTTPServer` a `flash.submit(watch=True)` opens on the run's
+output_dir. It serves exactly four things and 404s everything else:
+
+  GET /             → the live run page (viewer.page.render(); polls /api/state)
+  GET /api/state    → state.collect(run_dir) as JSON (the live snapshot)
+  GET /docs, /docs/…→ static files under the packaged _docs/ dir, else 404
+  (anything else)   → 404
+
+Threading, not a single-thread loop: the page polls /api/state every couple
+of seconds while a human also clicks around; a blocking handler would stall
+one request behind another. Everything is bound to 127.0.0.1 — this is a
+personal, local viewer, never a network service.
+"""
+
+from __future__ import annotations
+
+import json
+import mimetypes
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.parse import urlparse
+
+from flashruntime.viewer.page import render as render_page
+from flashruntime.viewer.state import collect
+
+# Packaged docs live here once Task 8's builder has run; absent in a fresh
+# checkout, which is why /docs degrades to an honest "docs not built" 404.
+_DEFAULT_DOCS_DIR = Path(__file__).parent / "_docs"
+
+# The run page is a static document (all liveness comes from its /api/state
+# polling), so render it once and reuse the bytes for every GET /.
+_PAGE_BYTES = render_page().encode()
+
+
+class _Handler(BaseHTTPRequestHandler):
+    """Routes GETs for one run. Reads its run_dir/docs_dir off the server
+    instance (`self.server`), so a single handler class serves any run."""
+
+    # Silence the default per-request stderr line: the user launched this from
+    # the same terminal their training logs to, and access-log spam would bury
+    # that output. (The viewer is deliberately quiet.)
+    def log_message(self, *args) -> None:  # noqa: D401
+        pass
+
+    def do_GET(self) -> None:
+        path = urlparse(self.path).path
+        if path == "/":
+            self._send(200, "text/html; charset=utf-8", _PAGE_BYTES)
+        elif path == "/api/state":
+            # collect() is total (never raises), so this branch cannot 500 on
+            # any on-disk state — it returns a snapshot or an {"error": ...}.
+            body = json.dumps(self.server.run_state()).encode()
+            self._send(200, "application/json; charset=utf-8", body)
+        elif path == "/docs" or path.startswith("/docs/"):
+            self._serve_docs(path)
+        else:
+            self._send(404, "text/plain; charset=utf-8", b"not found")
+
+    def _serve_docs(self, path: str) -> None:
+        docs_dir: Path = self.server.docs_dir
+        if not docs_dir.is_dir():
+            # The builder (Task 8) has not run in this checkout.
+            self._send(404, "text/plain; charset=utf-8", b"docs not built")
+            return
+        rel = path[len("/docs"):].lstrip("/") or "index.html"  # /docs and /docs/ → index.html
+        target = (docs_dir / rel).resolve()
+        # PATH-TRAVERSAL GUARD: `rel` comes straight from the request line, so
+        # a crafted "/docs/../../etc/passwd" would otherwise resolve OUTSIDE
+        # docs_dir. Reject anything whose resolved path is not under the docs
+        # root — serving files by request path without this check is a hole.
+        if not target.is_relative_to(docs_dir.resolve()) or not target.is_file():
+            self._send(404, "text/plain; charset=utf-8", b"not found")
+            return
+        ctype = mimetypes.guess_type(str(target))[0] or "application/octet-stream"
+        try:
+            body = target.read_bytes()
+        except OSError:
+            self._send(404, "text/plain; charset=utf-8", b"not found")
+            return
+        self._send(200, ctype, body)
+
+    def _send(self, code: int, ctype: str, body: bytes) -> None:
+        self.send_response(code)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+
+class RunViewerServer:
+    """Serve one run directory over HTTP on 127.0.0.1.
+
+    `port=0` (the default) asks the OS for a free port, so opening a viewer
+    never collides with another run's. `.start()` binds, spawns a daemon
+    serve loop, and returns the URL; `.stop()` shuts the loop down and closes
+    the socket (frees the port). `docs_dir` defaults to the packaged _docs/
+    but is injectable — the traversal guard and the docs routes are only
+    testable against a real directory, and the package's own _docs/ does not
+    exist until Task 8 builds it.
+    """
+
+    def __init__(self, run_dir: Path, port: int = 0, docs_dir: Path | None = None):
+        self.run_dir = Path(run_dir)
+        self.docs_dir = Path(docs_dir) if docs_dir is not None else _DEFAULT_DOCS_DIR
+        self._requested_port = port
+        self._httpd: ThreadingHTTPServer | None = None
+        self._thread: threading.Thread | None = None
+        self.url: str | None = None
+
+    def start(self) -> str:
+        httpd = ThreadingHTTPServer(("127.0.0.1", self._requested_port), _Handler)
+        # Hand the handler its per-server config. run_state is a bound method
+        # so each request re-reads the run dir fresh (a live snapshot).
+        httpd.run_state = lambda: collect(self.run_dir)  # type: ignore[attr-defined]
+        httpd.docs_dir = self.docs_dir  # type: ignore[attr-defined]
+        httpd.daemon_threads = True  # worker threads never block interpreter exit
+        self._httpd = httpd
+        port = httpd.server_address[1]  # the OS-assigned port when we asked for 0
+        self._thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+        self._thread.start()
+        self.url = f"http://127.0.0.1:{port}"
+        return self.url
+
+    def stop(self) -> None:
+        if self._httpd is not None:
+            self._httpd.shutdown()  # stop serve_forever
+            self._httpd.server_close()  # close the listening socket → port freed
+            self._httpd = None
+        if self._thread is not None:
+            self._thread.join(timeout=2)
+            self._thread = None

flashruntime/viewer/state.py

@@ -0,0 +1,250 @@
+"""state.collect(run_dir) — assemble the /api/state snapshot from disk.
+
+WHY EXCEPTION-SAFETY IS THE WHOLE POINT: this reads a *live* run's directory
+while the SDK is still writing into it. metrics.jsonl grows an unterminated
+last line between our read and the writer's flush; a manifest may be
+half-written; an attempt's output_dir may not exist yet; a file may be
+unreadable. A viewer that raised on any of these would turn "someone glanced
+at the run" into "the run's dashboard 500'd" — and worse, invite the reflex
+to make the SDK slow down or lock for the reader. So collect() is READ-ONLY
+and TOTAL: every disk hazard degrades to a partial snapshot (an empty
+section, a skipped record, an `"error"` string) and NEVER an exception. The
+run's story is never interrupted by being watched.
+
+Data model: pass the parsed `viewer_v1` run.json through verbatim, then
+ENRICH it — each attempt gains its own `metrics` (metrics.jsonl tail) and
+`log_tail` (launcher.log tail), `telemetry` (sampler tail) and `ranks`
+(per-rank heartbeats), read from that attempt's `output_dir`, and a
+top-level `checkpoints` list is assembled from the job ckpt roots (each root
+is the sibling `ckpt/` dir of an attempt dir). A top-level `monitor` object
+carries the newest telemetry sample (machine stats + process tree).
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import time
+from pathlib import Path
+
+_METRICS_TAIL = 500  # last N points per attempt — enough for a loss curve, bounded memory
+_LOG_TAIL = 100  # last N lines of launcher.log — a glance at what the process last said
+_TELEMETRY_TAIL = 150  # last N sampler ticks — a ~5 min usage chart at 2 s period
+
+# The tail window we seek-and-read from the END of a growing log. WHY BOUNDED:
+# the viewer re-reads these files every ~2 s against a LIVE run; a chatty
+# trainer's launcher.log / metrics.jsonl can reach multiple GB, and slurping
+# the whole file each poll would blow memory and thrash the disk against the
+# writer — the very interference this module promises not to cause. 256 KiB
+# comfortably holds the last 500 metrics records (~0.5 KiB each) or 100 log
+# lines of any sane width, so the wanted tail always lands inside one window.
+_TAIL_WINDOW = 256 * 1024
+
+
+def collect(run_dir: Path) -> dict:
+    """Read `<run_dir>/run.json` (the viewer_v1 contract) and return the
+    enriched /api/state dict. Never raises: an outer backstop converts any
+    unforeseen failure into `{"error": ...}` so a viewer can never crash a
+    live run — see the module docstring."""
+    try:
+        return _collect(Path(run_dir))
+    except Exception as exc:  # noqa: BLE001 — total by contract; see module docstring
+        return {"error": f"viewer snapshot failed: {exc!r}"}
+
+
+def _collect(run_dir: Path) -> dict:
+    run_json = run_dir / "run.json"
+    try:
+        raw = run_json.read_text(errors="replace")
+    except OSError as exc:
+        return {"error": f"run.json not found: {exc}"}
+    try:
+        doc = json.loads(raw)
+    except ValueError as exc:
+        # A torn read (writer mid-os.replace is atomic, but a truncated file
+        # from any other cause still lands here) yields an error, not a crash.
+        return {"error": f"run.json unreadable (torn or invalid JSON): {exc}"}
+    contract = doc.get("contract") if isinstance(doc, dict) else None
+    if contract != "viewer_v1":
+        # Fail closed on an unknown contract: a future/foreign schema is not
+        # something this version can honestly render, so say so rather than
+        # guess at fields that may have moved.
+        return {"error": f"unknown run.json contract: {contract!r}"}
+
+    # Enrich each attempt with its own metrics + log tail (both live in the
+    # attempt's output_dir), and collect the distinct job ckpt roots as we go.
+    ckpt_roots: dict[str, Path] = {}
+    enriched: list[dict] = []
+    for attempt in doc.get("attempts") or []:
+        row = dict(attempt)
+        out = row.get("output_dir")
+        attempt_dir = Path(out) if out else None
+        row["metrics"] = _metrics_tail(attempt_dir) if attempt_dir else []
+        row["log_tail"] = _log_tail(attempt_dir) if attempt_dir else ""
+        row["telemetry"] = _telemetry_tail(attempt_dir) if attempt_dir else []
+        row["ranks"] = _ranks(attempt_dir) if attempt_dir else []
+        enriched.append(row)
+        if attempt_dir is not None:
+            # The job ckpt root is the sibling `ckpt/` dir of the attempt dir
+            # (`<run>/<job>/<attempt>` → `<run>/<job>/ckpt`); restarts share a
+            # job_id, so many attempts map to one root — dedupe by path.
+            root = attempt_dir.parent / "ckpt"
+            ckpt_roots[str(root)] = root
+    doc["attempts"] = enriched
+
+    manifests: list[dict] = []
+    for root in ckpt_roots.values():
+        try:
+            manifests.extend(_manifests_for_root(root))
+        except Exception:  # noqa: BLE001 — one bad root never drops the others
+            continue
+    manifests.sort(key=lambda m: (m.get("job_id", ""), m.get("step", 0)))
+    doc["checkpoints"] = manifests
+
+    # The newest telemetry sample anywhere becomes the machine-level `monitor`
+    # object (KPI tiles + machine node read it). Newest-by-ts, not "the
+    # running attempt's", so a just-finished run still shows its last stats.
+    monitor = None
+    for row in enriched:
+        tel = row.get("telemetry") or []
+        cand = tel[-1] if tel else None
+        if isinstance(cand, dict) and (monitor is None or cand.get("ts", 0) > monitor.get("ts", 0)):
+            monitor = cand
+    doc["monitor"] = monitor
+    return doc
+
+
+def _tail_window_lines(path: Path) -> list[str]:
+    """Return the whole lines contained in the last `_TAIL_WINDOW` bytes of
+    `path` — a bounded seek+read that NEVER loads the whole file (see the
+    `_TAIL_WINDOW` note). Total: any OSError (missing / dir gone / unreadable)
+    degrades to `[]`, never an exception."""
+    try:
+        with path.open("rb") as fh:
+            fh.seek(0, io.SEEK_END)
+            size = fh.tell()
+            # Read a window ending at EOF rather than from byte 0: on a
+            # multi-GB log this is one bounded read, not a full slurp. When the
+            # file is smaller than the window, start clamps to 0 (whole file).
+            start = max(0, size - _TAIL_WINDOW)
+            fh.seek(start)
+            window = fh.read()
+    except OSError:
+        return []
+    lines = window.decode("utf-8", errors="replace").splitlines()
+    # If we began mid-file (start > 0), the first "line" is a torn fragment of
+    # the record straddling the window's start byte — drop it so only whole
+    # lines escape (a partial JSON metrics line would skip on parse anyway).
+    if start > 0 and lines:
+        lines = lines[1:]
+    return lines
+
+
+def _metrics_tail(attempt_dir: Path, limit: int = _METRICS_TAIL) -> list[dict]:
+    """Last `limit` JSON records from `metrics.jsonl`, keys passed through
+    as-is. A half-written final line (the writer appended between our read and
+    its newline) fails to parse and is skipped — the earlier points survive.
+    Reads only the bounded tail window, never the whole file."""
+    records: list[dict] = []
+    for line in _tail_window_lines(attempt_dir / "metrics.jsonl"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rec = json.loads(line)
+        except ValueError:
+            continue  # torn/partial line — skip it, keep the rest
+        if isinstance(rec, dict):
+            records.append(rec)
+    return records[-limit:]
+
+
+def _telemetry_tail(attempt_dir: Path, limit: int = _TELEMETRY_TAIL) -> list[dict]:
+    """Last `limit` sampler ticks from `telemetry.jsonl` — the same bounded
+    tail-window + skip-torn-lines discipline as `_metrics_tail` (the sampler
+    appends live while we read)."""
+    records: list[dict] = []
+    for line in _tail_window_lines(attempt_dir / "telemetry.jsonl"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rec = json.loads(line)
+        except ValueError:
+            continue  # torn/partial line — skip it, keep the rest
+        if isinstance(rec, dict):
+            records.append(rec)
+    return records[-limit:]
+
+
+def _ranks(attempt_dir: Path) -> list[dict]:
+    """Every parseable `ranks/rank-*.json` heartbeat, sorted by rank. A torn
+    file (a rank mid-os.replace cannot tear, but a foreign writer could) is
+    skipped; a missing dir is simply 'not instrumented' — []."""
+    try:
+        paths = sorted((attempt_dir / "ranks").glob("rank-*.json"))
+    except OSError:
+        return []
+    out: list[dict] = []
+    for p in paths:
+        try:
+            rec = json.loads(p.read_text())
+        except (OSError, ValueError):
+            continue
+        if isinstance(rec, dict):
+            out.append(rec)
+    out.sort(key=lambda r: r.get("rank", 0) if isinstance(r.get("rank", 0), int) else 0)
+    return out
+
+
+def _log_tail(attempt_dir: Path, tail_lines: int = _LOG_TAIL) -> str:
+    """Last `tail_lines` lines of `launcher.log`, or "" if unreadable. Reads
+    only the bounded tail window, never the whole file."""
+    return "\n".join(_tail_window_lines(attempt_dir / "launcher.log")[-tail_lines:])
+
+
+def _manifests_for_root(ckpt_root: Path) -> list[dict]:
+    """List every `step-*/manifest.json` under one job ckpt root, each
+    re-verified against disk. Reuses `flashruntime.checkpoint.local`'s
+    hashing (`verify_manifest`) and its picker (`latest_valid_manifest`) so
+    the "is this checkpoint safe" logic is never duplicated here — the viewer
+    only *reads* checkpoints, it must agree exactly with what recovery would
+    restore. Each entry: step, re-verified validation, part count, age, and
+    whether it is the one recovery would pick (`latest_valid`)."""
+    from flashruntime.checkpoint.local import (
+        MANIFEST_NAME,
+        latest_valid_manifest,
+        verify_manifest,
+    )
+    from flashruntime.protocol.v1alpha1 import CheckpointManifest
+
+    ckpt_root = Path(ckpt_root)
+    if not ckpt_root.is_dir():
+        return []
+    latest = latest_valid_manifest(ckpt_root)  # the manifest recovery would restore
+    latest_step = latest.step if latest else None
+    out: list[dict] = []
+    for mf_path in sorted(ckpt_root.glob(f"step-*/{MANIFEST_NAME}")):
+        try:
+            manifest = CheckpointManifest.model_validate_json(mf_path.read_text())
+        except (OSError, ValueError):
+            continue  # torn/invalid manifest — skip, keep scanning (never crash)
+        valid = verify_manifest(manifest, mf_path.parent)
+        try:
+            age_s: float | None = time.time() - manifest.created.timestamp()
+        except Exception:  # noqa: BLE001 — a weird created value must not crash the scan
+            age_s = None
+        out.append(
+            {
+                "job_id": manifest.job_id,
+                "step": manifest.step,
+                # Report the RE-VERIFIED state, not the manifest's stored claim:
+                # a part corrupted after writing makes a "hash_verified" manifest
+                # actually invalid, and the viewer must show that truth.
+                "validation": "hash_verified" if valid else "invalid",
+                "parts": len(manifest.parts),
+                "age_s": age_s,
+                "latest_valid": bool(valid and manifest.step == latest_step),
+            }
+        )
+    return out

flashruntime/workloads/__init__.py

@@ -0,0 +1,6 @@
+"""Workload descriptions: WHAT the user wants run (four-axes rule: this is
+the axis-zero input the recipes/strategies/launchers axes consume)."""
+
+from flashruntime.workloads.command import CommandWorkload, OutputSpec, Source, to_jobspec
+
+__all__ = ["CommandWorkload", "OutputSpec", "Source", "to_jobspec"]

flashruntime/workloads/command.py

@@ -0,0 +1,127 @@
+"""User-facing description of a "bring your own code" workload.
+
+A CommandWorkload names WHAT to run (a command in a source directory,
+optionally in a pinned image) and what FlashRuntime should do around it
+(inputs, outputs, isolation, Mode A fan-out). It never describes HOW
+distributed math happens — that belongs to the user's code and its
+framework (ADR-0003: FlashRuntime operates jobs, it does not train).
+
+Pydantic-only: importing this module must never require torch, sklearn,
+kubernetes, or fastapi — it is part of the clean core.
+"""
+
+from __future__ import annotations
+
+import shlex
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+from flashruntime.protocol.plan_v1alpha1 import CheckpointPolicy
+from flashruntime.protocol.v1alpha1 import (
+    ExecutionSpec,
+    ImageSpec,
+    IsolationSpec,
+    JobMetadata,
+    JobSpec,
+    JobSpecInner,
+    WorkloadSpec,
+)
+from flashruntime.providers import Requirements
+
+
+class Source(BaseModel):
+    """Where the user's code lives. v1 executes from a local directory;
+    `git_revision` is reserved for remote packaging (spec §10 follow-up)."""
+
+    path: str = "."
+    git_revision: str | None = None
+
+
+class OutputSpec(BaseModel):
+    """What to keep after a run. `collect` globs are resolved against the
+    script's working directory; `primary_metric` names the metrics.json key
+    Run.best_trial() ranks by."""
+
+    prefix: str = "artifact://jobs/{job_id}/"
+    collect: list[str] = Field(default_factory=lambda: ["metrics.json"])
+    primary_metric: str | None = None
+    maximize: bool = True
+
+
+class CommandWorkload(BaseModel):
+    """One command, operated by FlashRuntime.
+
+    `command` may be a shell-style string (shlex-split, never shell=True —
+    pipes need an explicit `bash -c "..."`) or an argv list. `{name}`
+    placeholders are filled per `task_params` entry for Mode A fan-out.
+    """
+
+    command: str | list[str]
+    source: Source = Field(default_factory=Source)
+    image: ImageSpec | None = None
+    env: dict[str, str] = Field(default_factory=dict)
+    inputs: dict[str, str] = Field(default_factory=dict)
+    outputs: OutputSpec = Field(default_factory=OutputSpec)
+    resources: Requirements = Field(default_factory=Requirements)
+    isolation: IsolationSpec = Field(default_factory=IsolationSpec)
+    mode: Literal["auto", "local", "independent_tasks", "coordinated"] = "auto"
+    checkpoint: CheckpointPolicy | None = None
+    task_params: list[dict] | None = None
+
+    @field_validator("inputs")
+    @classmethod
+    def _artifact_scheme(cls, v: dict[str, str]) -> dict[str, str]:
+        for name, uri in v.items():
+            if not str(uri).startswith("artifact://"):
+                raise ValueError(f"input '{name}' must be an artifact:// URI, got {uri!r}")
+        return v
+
+    def argv(self, params: dict | None = None) -> list[str]:
+        """Exec-ready argv. `params` fills `{name}` placeholders; a
+        placeholder with no matching param raises KeyError (a silent empty
+        substitution would corrupt the command)."""
+        tokens = shlex.split(self.command) if isinstance(self.command, str) else list(self.command)
+        if params:
+            tokens = [t.format(**params) for t in tokens]
+        return tokens
+
+    def resolved_mode(self) -> str:
+        """Deterministic `auto` resolution (spec §4.1): fan-out params ⇒
+        independent_tasks; a multi-process launcher command ⇒ coordinated;
+        else local. An explicit `mode` always wins."""
+        if self.mode != "auto":
+            return self.mode
+        if self.task_params:
+            return "independent_tasks"
+        tokens = self.argv()
+        if tokens and tokens[0] in ("torchrun", "accelerate"):
+            return "coordinated"
+        return "local"
+
+
+def to_jobspec(workload: CommandWorkload, name: str, image: ImageSpec | None = None) -> JobSpec:
+    """Wire form for the coordinator: JobSpec{execution.backend: leases,
+    workload.type: "command"}. A pinned image is required — remote runs
+    must be reproducible (the schema already rejects 'latest')."""
+    img = image or workload.image
+    if img is None:
+        raise ValueError("a pinned image is required to submit a command workload to the service")
+    parameters: dict = {
+        "command": workload.argv(),  # normalized argv, placeholders intact
+        "env": dict(workload.env),
+        "inputs": dict(workload.inputs),
+    }
+    if workload.task_params is not None:
+        parameters["task_params"] = workload.task_params
+    if workload.checkpoint is not None:
+        parameters["checkpoint"] = workload.checkpoint.model_dump()
+    return JobSpec(
+        metadata=JobMetadata(name=name),
+        spec=JobSpecInner(
+            execution=ExecutionSpec(backend="leases"),
+            image=img,
+            workload=WorkloadSpec(type="command", parameters=parameters),
+            isolation=workload.isolation,
+        ),
+    )