flashruntime 0.3.0__py3-none-any.whl

flashruntime/service/cli.py

@@ -0,0 +1,167 @@
+"""Thin CLI over the FlashRuntime API — plus the offline planner and the
+local "bring your own code" front door.
+
+  flashruntime plan path/to/plan.yaml [--json]   # no API/cluster needed
+  flashruntime submit "python train.py" [--source DIR] [--task-params JSON] \
+      [--max-restarts N] [--output-dir DIR] [--watch|--no-watch]  # local, no API
+  flashruntime submit-spec path/to/job.yaml [--api URL]  # POST a JobSpec to the coordinator
+  flashruntime status <job-id>
+  flashruntime events <job-id>
+  flashruntime logs <job-id>
+  flashruntime cancel <job-id>
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+
+def _api(args) -> str:
+    return args.api or os.environ.get("FLASHML_RUNTIME_API", "http://localhost:8100")
+
+
+def _plan(args) -> int:
+    """Run the offline strategy planner: file → PlanRequest → PlanReport."""
+    from flashruntime.planner import plan, render
+    from flashruntime.protocol.plan_v1alpha1 import PlanRequest
+
+    if args.request_file.endswith((".yaml", ".yml")):
+        try:
+            import yaml
+        except ImportError:
+            print("pyyaml is required for YAML input: pip install pyyaml", file=sys.stderr)
+            return 2
+        with open(args.request_file) as f:
+            raw = yaml.safe_load(f)
+    else:
+        with open(args.request_file) as f:
+            raw = json.load(f)
+
+    try:
+        request = PlanRequest.model_validate(raw)
+    except Exception as exc:  # pydantic ValidationError — show it plainly
+        print(f"invalid PlanRequest: {exc}", file=sys.stderr)
+        return 1
+
+    report = plan(request)
+    if args.json:
+        print(report.model_dump_json(indent=2, exclude_none=True))
+    else:
+        print(render(report))
+    return 0 if report.selected is not None else 3
+
+
+def _submit(args) -> int:
+    """Build a CommandWorkload and run it locally via the SDK, then print a
+    human summary. Exit 0 iff SUCCEEDED. The SDK is imported here (not at
+    module top) so cli.py stays cheap on coordinator-only installs where the
+    other subcommands only need httpx."""
+    from flashruntime.sdk import submit
+    from flashruntime.workloads.command import CommandWorkload, Source
+
+    try:
+        task_params = json.loads(args.task_params) if args.task_params else None
+    except json.JSONDecodeError as exc:  # a bad --task-params must not traceback
+        print(f"--task-params is not valid JSON: {exc}", file=sys.stderr)
+        return 2
+
+    workload = CommandWorkload(
+        command=args.cmd,
+        source=Source(path=args.source),
+        task_params=task_params,
+    )
+
+    # --watch is a tri-state: True (--watch) / False (--no-watch) / None
+    # (unset → submit() decides by TTY, off in CI). When on, submit() opens
+    # the live viewer and prints its URL, so the CLI just passes it through.
+    run = submit(
+        workload,
+        output_dir=args.output_dir,
+        max_restarts=args.max_restarts,
+        watch=args.watch,
+    )
+
+    print(f"state:     {run.state.value}")
+    print(f"trials:    {len(run.trials)}")
+    if workload.outputs.primary_metric:
+        best = run.best_trial()
+        print(f"best:      {best}" if best else "best:      (no trial reported the metric)")
+    print(f"output:    {run.output_dir}")
+    return 0 if run.state.value == "SUCCEEDED" else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="flashruntime")
+    parser.add_argument("--api", help="FlashRuntime API base URL")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_plan = sub.add_parser("plan", help="evaluate a PlanRequest offline and print the strategy")
+    p_plan.add_argument("request_file", help="PlanRequest as .yaml or .json")
+    p_plan.add_argument("--json", action="store_true", help="emit the full PlanReport as JSON")
+
+    p_submit = sub.add_parser("submit", help="run a command workload locally (no API needed)")
+    # dest is `cmd`, not `command`: the subparsers dest is already `command`
+    # (the subcommand name), and a positional named `command` would clobber it.
+    p_submit.add_argument("cmd", metavar="CMD", help="the command to run, e.g. 'python train.py --lr {lr}'")
+    p_submit.add_argument("--source", default=".", help="directory holding the user's code")
+    p_submit.add_argument("--task-params", help="JSON list of param dicts for Mode A fan-out")
+    p_submit.add_argument("--max-restarts", type=int, default=0, help="automatic recovery budget")
+    p_submit.add_argument("--output-dir", help="where run.json and artifacts land (default: temp dir)")
+    p_submit.add_argument(
+        "--watch",
+        action=argparse.BooleanOptionalAction,
+        default=None,  # None ⇒ decide by TTY in _submit (never block/open a viewer in CI)
+        help="open the live viewer (default: on at a terminal, off in pipes/CI)",
+    )
+
+    p_submit_spec = sub.add_parser(
+        "submit-spec",
+        help=(
+            "POST a JobSpec YAML to the coordinator — was `submit` before 0.1.0; "
+            "renamed when `submit` became the local-workload front door"
+        ),
+    )
+    p_submit_spec.add_argument("spec_file")
+    for name in ("status", "events", "logs", "cancel"):
+        p = sub.add_parser(name)
+        p.add_argument("job_id")
+
+    args = parser.parse_args(argv)
+
+    if args.command == "plan":
+        return _plan(args)
+    if args.command == "submit":
+        return _submit(args)
+
+    import httpx
+
+    base = _api(args)
+    try:
+        if args.command == "submit-spec":
+            import yaml
+
+            with open(args.spec_file) as f:
+                spec = yaml.safe_load(f)
+            r = httpx.post(f"{base}/v1alpha1/jobs", json=spec, timeout=60)
+        elif args.command == "cancel":
+            r = httpx.post(f"{base}/v1alpha1/jobs/{args.job_id}/cancel", timeout=60)
+        elif args.command == "status":
+            r = httpx.get(f"{base}/v1alpha1/jobs/{args.job_id}", timeout=30)
+        else:
+            r = httpx.get(f"{base}/v1alpha1/jobs/{args.job_id}/{args.command}", timeout=30)
+    except httpx.ConnectError as exc:
+        print(f"cannot reach FlashRuntime API at {base}: {exc}", file=sys.stderr)
+        return 2
+
+    if r.status_code >= 400:
+        print(f"error {r.status_code}: {r.text}", file=sys.stderr)
+        return 1
+    print(json.dumps(r.json(), indent=2, default=str))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

flashruntime/service/dashboard.py

@@ -0,0 +1,193 @@
+"""The coordinator's built-in status page.
+
+One self-contained HTML page at GET / — no build step, no external assets,
+no framework: it polls the same JSON API everything else uses
+(/v1alpha1/nodes, /v1alpha1/jobs, /v1alpha1/jobs/{id}/{tasks,events,artifacts})
+every 2 seconds. This is the self-hosted profile's window into the system;
+FlashML Cloud's Next.js dashboard is the managed superset of the same data.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from fastapi.responses import HTMLResponse
+
+from flashruntime.viewer.page import TOKENS
+
+_RAW_PAGE = """<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>FlashRuntime — local coordinator</title>
+<style>
+  :root { color-scheme: dark; }
+  * { box-sizing: border-box; margin: 0; }
+  body { font: 13px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace;
+         background: #0d1117; color: #c9d1d9; padding: 24px; }
+  h1 { font-size: 15px; margin-bottom: 4px; color: #e6edf3; }
+  h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .08em;
+       color: #8b949e; margin: 24px 0 8px; }
+  .sub { color: #8b949e; margin-bottom: 12px; }
+  table { border-collapse: collapse; width: 100%; }
+  th, td { text-align: left; padding: 5px 12px 5px 0; border-bottom: 1px solid #21262d;
+           vertical-align: top; white-space: nowrap; }
+  th { color: #8b949e; font-weight: normal; }
+  td.wrap { white-space: normal; }
+  .dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%;
+         margin-right: 6px; }
+  .on  { background: #3fb950; } .off { background: #f85149; }
+  .st-RUNNING { color: #58a6ff; } .st-SUCCEEDED, .st-COMPLETED { color: #3fb950; }
+  .st-FAILED { color: #f85149; } .st-PENDING { color: #8b949e; }
+  .st-LEASED { color: #d29922; } .st-RECOVERING { color: #d29922; }
+  .st-CANCELLED { color: #8b949e; }
+  tr.job { cursor: pointer; } tr.job:hover td { background: #161b22; }
+  tr.sel td { background: #161b22; }
+  #events { max-height: 320px; overflow-y: auto; }
+  .ev-time { color: #8b949e; margin-right: 8px; }
+  .ev-type { color: #d2a8ff; margin-right: 8px; }
+  a { color: #58a6ff; text-decoration: none; }
+  .cols { display: flex; gap: 40px; flex-wrap: wrap; }
+  .cols > div { flex: 1 1 420px; min-width: 0; overflow-x: auto; }
+  .muted { color: #8b949e; }
+</style>
+</head>
+<body>
+<h1>FlashRuntime — local coordinator</h1>
+<div class="sub" id="health">connecting…</div>
+
+<h2>Nodes</h2>
+<table><thead><tr><th></th><th>node</th><th>host</th><th>env</th><th>cpu</th>
+<th>ram</th><th>accepted tasks</th><th>last heartbeat</th></tr></thead>
+<tbody id="nodes"><tr><td colspan="8" class="muted">no nodes registered</td></tr></tbody></table>
+
+<h2>Jobs</h2>
+<table><thead><tr><th>job</th><th>name</th><th>backend</th><th>state</th>
+<th>tasks</th><th>created</th></tr></thead>
+<tbody id="jobs"><tr><td colspan="6" class="muted">no jobs yet</td></tr></tbody></table>
+
+<div class="cols">
+  <div>
+    <h2>Tasks <span class="muted" id="selJob"></span></h2>
+    <table><thead><tr><th>task</th><th>state</th><th>attempts</th><th>node</th></tr></thead>
+    <tbody id="tasks"><tr><td colspan="4" class="muted">select a job</td></tr></tbody></table>
+    <h2>Artifacts</h2>
+    <div id="artifacts" class="muted">select a job</div>
+  </div>
+  <div>
+    <h2>Events</h2>
+    <div id="events" class="muted">select a job</div>
+  </div>
+</div>
+
+<script>
+const $ = id => document.getElementById(id);
+let selected = null, taskCache = {};
+
+const esc = s => String(s ?? "").replace(/[&<>"]/g, c =>
+  ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;"}[c]));
+const st = s => `<span class="st-${esc(s)}">${esc(s)}</span>`;
+const t = iso => iso ? new Date(iso).toLocaleTimeString() : "";
+
+async function j(path) { const r = await fetch(path); if (!r.ok) throw r; return r.json(); }
+
+async function tick() {
+  try {
+    const h = await j("/healthz");
+    $("health").textContent = `profile: ${h.profile} · ${new Date().toLocaleTimeString()}`;
+  } catch { $("health").textContent = "coordinator unreachable"; return; }
+
+  const nodes = await j("/v1alpha1/nodes");
+  $("nodes").innerHTML = nodes.length ? nodes.map(n => `<tr>
+    <td><span class="dot ${n.online ? "on" : "off"}"></span></td>
+    <td>${esc(n.node_id)}</td><td>${esc(n.hostname)}</td><td>${esc(n.environment)}</td>
+    <td>${n.capabilities.cpu_cores ?? "?"}</td>
+    <td>${n.capabilities.memory_bytes ? (n.capabilities.memory_bytes/1e9).toFixed(1)+" GB" : "?"}</td>
+    <td>${n.accepted_tasks}</td><td>${n.last_heartbeat_age_s}s ago</td></tr>`).join("")
+    : `<tr><td colspan="8" class="muted">no nodes registered</td></tr>`;
+
+  const jobs = await j("/v1alpha1/jobs");
+  for (const job of jobs) {
+    if (job.backend === "leases") {
+      try { taskCache[job.job_id] = await j(`/v1alpha1/jobs/${job.job_id}/tasks`); } catch {}
+    }
+  }
+  $("jobs").innerHTML = jobs.length ? jobs.map(job => {
+    const tasks = taskCache[job.job_id] || [];
+    const done = tasks.filter(x => x.state === "COMPLETED").length;
+    const summary = tasks.length ? `${done}/${tasks.length} completed` : "—";
+    return `<tr class="job ${job.job_id === selected ? "sel" : ""}" onclick="sel('${job.job_id}')">
+      <td>${esc(job.job_id)}</td><td>${esc(job.spec.metadata.name)}</td>
+      <td>${esc(job.backend)}</td><td>${st(job.state)}</td>
+      <td>${summary}</td><td>${t(job.created_at)}</td></tr>`;
+  }).join("") : `<tr><td colspan="6" class="muted">no jobs yet</td></tr>`;
+
+  if (selected) await detail(selected);
+}
+
+async function detail(id) {
+  $("selJob").textContent = "· " + id;
+  const tasks = taskCache[id] || [];
+  $("tasks").innerHTML = tasks.length ? tasks.map(x => `<tr>
+    <td>${esc(x.task_id)}</td><td>${st(x.state)}</td>
+    <td>${x.attempts}/${x.max_attempts}</td><td>${esc(x.node_id ?? "—")}</td></tr>`).join("")
+    : `<tr><td colspan="4" class="muted">no leased tasks (ray-backend job?)</td></tr>`;
+
+  try {
+    const events = await j(`/v1alpha1/jobs/${id}/events`);
+    $("events").innerHTML = events.slice().reverse().map(e =>
+      `<div><span class="ev-time">${t(e.timestamp)}</span>` +
+      `<span class="ev-type">${esc(e.type)}</span>${esc(e.message)}</div>`).join("");
+  } catch {}
+  try {
+    const arts = await j(`/v1alpha1/jobs/${id}/artifacts`);
+    $("artifacts").innerHTML = arts.length ? arts.map(a =>
+      `<div><a href="/v1alpha1/artifacts/${esc(a.key)}" target="_blank">${esc(a.key)}</a>` +
+      ` <span class="muted">(${a.size_bytes} B)</span></div>`).join("") : "none yet";
+  } catch {}
+}
+
+function sel(id) { selected = id; tick(); }
+tick(); setInterval(tick, 2000);
+</script>
+</body>
+</html>"""
+
+
+# Palette alignment: the dashboard was built with ad-hoc GitHub-dark hexes;
+# map each to the shared token in `viewer.page.TOKENS` so this page and the
+# run viewer read as one house style. VALUES only — the page structure is
+# untouched. The neutrals map to themselves; the accents pick up the oklch
+# tokens (cyan running, green ok, amber warn, red fail, violet checkpoints).
+_PALETTE = {
+    "#0d1117": TOKENS["bg"],
+    "#161b22": TOKENS["panel"],
+    "#21262d": TOKENS["border"],
+    "#c9d1d9": TOKENS["text"],
+    "#e6edf3": TOKENS["text_bright"],
+    "#8b949e": TOKENS["muted"],
+    "#3fb950": TOKENS["ok"],  # on / SUCCEEDED / COMPLETED
+    "#f85149": TOKENS["fail"],  # off / FAILED
+    "#58a6ff": TOKENS["running"],  # RUNNING / links
+    "#d29922": TOKENS["warn"],  # LEASED / RECOVERING
+    "#d2a8ff": TOKENS["ckpt"],  # event type accent
+}
+
+
+def _recolor(page: str) -> str:
+    """Swap every ad-hoc hex for its shared token (single source of truth)."""
+    for old, new in _PALETTE.items():
+        page = page.replace(old, new)
+    return page
+
+
+_PAGE = _recolor(_RAW_PAGE)
+
+
+def build_router() -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/", include_in_schema=False)
+    async def index() -> HTMLResponse:
+        return HTMLResponse(_PAGE)
+
+    return router

flashruntime/service/ledger.py

@@ -0,0 +1,101 @@
+"""Append-only job/event ledger on SQLite.
+
+POC-scale persistence: one file, synchronous sqlite3 behind a lock, called
+via asyncio.to_thread. Postgres is a flashml-cloud concern; the runtime's
+ledger only needs durability and ordering.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+
+from flashruntime.protocol.v1alpha1 import (
+    ArtifactRecord,
+    Event,
+    JobRecord,
+    JobState,
+)
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS jobs (
+    job_id TEXT PRIMARY KEY,
+    record TEXT NOT NULL,
+    state TEXT NOT NULL,
+    created_at TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS events (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    job_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    timestamp TEXT NOT NULL,
+    payload TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_events_job ON events (job_id, seq);
+"""
+
+
+class Ledger:
+    def __init__(self, path: str | Path):
+        self._path = str(path)
+        self._lock = threading.Lock()
+        with self._conn() as conn:
+            conn.executescript(_SCHEMA)
+
+    def _conn(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    # -- jobs --------------------------------------------------------------
+
+    def upsert_job(self, job: JobRecord) -> None:
+        with self._lock, self._conn() as conn:
+            conn.execute(
+                "INSERT INTO jobs (job_id, record, state, created_at) VALUES (?,?,?,?) "
+                "ON CONFLICT(job_id) DO UPDATE SET record=excluded.record, state=excluded.state",
+                (
+                    job.job_id,
+                    job.model_dump_json(),
+                    job.state.value,
+                    job.created_at.isoformat(),
+                ),
+            )
+
+    def get_job(self, job_id: str) -> JobRecord | None:
+        with self._lock, self._conn() as conn:
+            row = conn.execute(
+                "SELECT record FROM jobs WHERE job_id=?", (job_id,)
+            ).fetchone()
+        return JobRecord.model_validate_json(row["record"]) if row else None
+
+    def list_jobs(self) -> list[JobRecord]:
+        with self._lock, self._conn() as conn:
+            rows = conn.execute(
+                "SELECT record FROM jobs ORDER BY created_at DESC"
+            ).fetchall()
+        return [JobRecord.model_validate_json(r["record"]) for r in rows]
+
+    # -- events ------------------------------------------------------------
+
+    def append_event(self, event: Event) -> None:
+        with self._lock, self._conn() as conn:
+            conn.execute(
+                "INSERT INTO events (job_id, type, timestamp, payload) VALUES (?,?,?,?)",
+                (
+                    event.job_id,
+                    event.type.value,
+                    event.timestamp.isoformat(),
+                    event.model_dump_json(),
+                ),
+            )
+
+    def events_for(self, job_id: str) -> list[Event]:
+        with self._lock, self._conn() as conn:
+            rows = conn.execute(
+                "SELECT payload FROM events WHERE job_id=? ORDER BY seq", (job_id,)
+            ).fetchall()
+        return [Event.model_validate_json(r["payload"]) for r in rows]