flashruntime 0.3.0__py3-none-any.whl

flashruntime/recovery/taxonomy.py

@@ -0,0 +1,91 @@
+"""Deterministic failure classification from raw signals.
+
+Turns what was *observed* (exit codes, heartbeat loss, NCCL messages, XID
+events, storage errors) into one typed `FailureClass`. Rules are
+precedence-ordered — the first matching class wins — and the order encodes
+root-cause priority: systemic evidence (correlated incidents, control-plane
+loss) beats node evidence beats process evidence, because a worker crash
+*during* a correlated incident is the incident, not the worker.
+
+No inference beyond the table. Ambiguity resolves to conservative classes
+(UNKNOWN classifies, policy decides what UNKNOWN deserves) rather than
+optimistic ones.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from flashruntime.protocol.v1alpha1 import FailureClass
+
+# Substrings that identify communication-layer failures in worker logs.
+# [assumption: covers the common NCCL/Gloo surfaces; extend from ledger data]
+_COMM_MARKERS = (
+    "nccl",
+    "rccl",
+    "gloo",
+    "rendezvous",
+    "allreduce",
+    "all_reduce",
+    "process group",
+)
+_STORAGE_MARKERS = ("s3", "oss", "minio", "object store", "storage timeout", "slowdown")
+
+
+@dataclass(frozen=True)
+class FailureSignals:
+    """The raw evidence available when something broke. All fields optional —
+    classification works from whatever was actually observed."""
+
+    exit_code: int | None = None
+    exit_deterministic: bool = False  # same error at the same step on retry
+    heartbeat_lost: bool = False
+    node_unreachable: bool = False
+    xid_event: bool = False  # NVIDIA XID / ECC / NVML health failure
+    driver_fault: bool = False
+    hash_mismatch: bool = False
+    validation_failed: bool = False
+    preemption_notice: bool = False
+    storage_errors: int = 0
+    control_plane_unreachable: bool = False
+    concurrent_failures: int = 1  # failures in the correlation window, incl. this one
+    message: str = ""
+
+    def _msg(self) -> str:
+        return self.message.lower()
+
+    def has_comm_marker(self) -> bool:
+        return any(m in self._msg() for m in _COMM_MARKERS)
+
+    def has_storage_marker(self) -> bool:
+        return any(m in self._msg() for m in _STORAGE_MARKERS)
+
+
+# How many failures inside the correlation window turn individual failures
+# into one systemic incident. [assumption: recalibrate from ledger data]
+CORRELATED_THRESHOLD = 3
+
+
+def classify(s: FailureSignals) -> FailureClass:
+    """Precedence-ordered classification. Systemic > node > process > app."""
+    if s.concurrent_failures >= CORRELATED_THRESHOLD:
+        return FailureClass.CORRELATED_INCIDENT
+    if s.control_plane_unreachable:
+        return FailureClass.CONTROL_PLANE_FAILURE
+    if s.preemption_notice:
+        return FailureClass.PREEMPTION
+    if s.xid_event or s.driver_fault:
+        return FailureClass.ACCELERATOR_FAILURE
+    if s.heartbeat_lost or s.node_unreachable:
+        return FailureClass.NODE_LOSS
+    if s.has_comm_marker():
+        return FailureClass.COMMUNICATION_ERROR
+    if s.storage_errors > 0 or s.has_storage_marker():
+        return FailureClass.STORAGE_TIMEOUT
+    if s.hash_mismatch or s.validation_failed:
+        return FailureClass.ARTIFACT_CORRUPTION
+    if s.exit_code is not None and s.exit_code != 0:
+        if s.exit_deterministic:
+            return FailureClass.APPLICATION_ERROR
+        return FailureClass.WORKER_CRASH
+    return FailureClass.UNKNOWN

flashruntime/scheduler/__init__.py

@@ -0,0 +1,170 @@
+"""Placement policy: which task should a claiming node receive?
+
+Today's live behavior is FIFO over PENDING tasks (`LeaseStore.next_pending`
+insertion order) — honest and correct while tasks are CPU-sized and
+homogeneous. `FifoPlacement` encodes exactly that behavior as the first
+concrete policy, so wiring this interface into the claim path changes
+nothing until a richer policy is chosen deliberately.
+
+This package becomes load-bearing when payloads carry resource
+requirements (vram_gb, min_cpus) and claims must filter by node
+capability — research item R9 in `HANDBOOK.md` §7. The master report's §9
+discipline applies: capability ("can it run this?"), reliability ("will it
+finish?") and trust ("may this data go there?") stay SEPARATE assessments;
+this interface handles capability filtering + preference ordering only.
+Reliability scoring waits for ledger volume; trust tiers are cloud policy.
+
+Integration point (when the time comes): `LeaseManager.claim` consults the
+policy instead of calling `next_pending` directly — the store keeps
+returning candidates in insertion order; the policy filters and picks.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from flashruntime.protocol.v1alpha1 import TaskSpec
+
+__all__ = ["PlacementPolicy", "FifoPlacement", "IsolationAwarePlacement"]
+
+#: A node as the policy sees it: the registry's view dict
+#: ({"node_id", "capabilities": {...}, ...}). Kept as a mapping (not a
+#: model) until the capability schema for placement stabilizes with R9.
+NodeView = dict[str, Any]
+
+
+class PlacementPolicy(ABC):
+    """Filter + order candidate tasks for one claiming node.
+
+    Contracts:
+    - `eligible` is a pure predicate — no I/O, no side effects; called
+      once per (task, node) per claim, so it must be cheap.
+    - `score` orders eligible tasks (higher = better for THIS node);
+      ties break by queue order (fairness/FIFO stays the default
+      tiebreak so starvation cannot be introduced accidentally).
+    - `choose` is a template method most policies should NOT override —
+      override `eligible`/`score` and keep the selection semantics shared.
+    - A policy must never *create* work or mutate tasks; it only selects.
+    """
+
+    @abstractmethod
+    def eligible(self, task: TaskSpec, node: NodeView) -> bool:
+        """May this node run this task at all? (capability gate — e.g.
+        payload wants vram_gb the node lacks → False). Unknown/absent
+        requirements must default to True: fail-open on *placement*,
+        because the executor's allowlists still fail-closed on *safety*."""
+
+    def score(self, task: TaskSpec, node: NodeView) -> float:
+        """Preference among eligible tasks for this node (higher wins).
+        Default 0.0 for all ⇒ pure FIFO. Examples later: prefer tasks
+        whose input artifacts this node already cached; prefer short
+        tasks for soon-to-drain nodes."""
+        return 0.0
+
+    def choose(self, pending: list[TaskSpec], node: NodeView) -> TaskSpec | None:
+        """Select the task to lease to `node` from queue-ordered `pending`.
+
+        Semantics: filter by `eligible`, take the max `score`, break ties
+        by earliest queue position. Returns None when nothing is eligible
+        (the claim endpoint answers 204, exactly like an empty queue).
+        """
+        best: TaskSpec | None = None
+        best_score = float("-inf")
+        for task in pending:  # queue order ⇒ first max wins ties
+            if not self.eligible(task, node):
+                continue
+            s = self.score(task, node)
+            if s > best_score:
+                best, best_score = task, s
+        return best
+
+
+class FifoPlacement(PlacementPolicy):
+    """The current system, as a policy: everything eligible, no
+    preference, first-come-first-served. Wiring this in is a pure
+    refactor — behavior is bit-identical to `next_pending`."""
+
+    def eligible(self, task: TaskSpec, node: NodeView) -> bool:
+        return True
+
+
+class IsolationAwarePlacement(PlacementPolicy):
+    """FIFO plus the one fail-closed capability gate the isolation contract
+    requires: a task whose payload demands sandboxed execution may only go
+    to a node advertising `sandbox_capable` — an ABSENT capability counts as
+    NOT capable (security-relevant fields fail closed, AGENTS.md rule 3).
+
+    The gate requires *true booleans*; type-confused values fail closed:
+
+    - A node is capable only when `sandbox_capable is True`. A truthy
+      stand-in (the JSON string ``"false"``, ``1``, ``"yes"``) does NOT
+      count as capable.
+    - The task's own `allowFallback` waives the requirement only when it is
+      exactly `True`; any other value (including the string ``"false"``)
+      does not waive.
+    - The isolation payload must be a mapping. If it is present but not a
+      dict (e.g. the bare string ``"sandboxed"``), the task is ineligible
+      everywhere — fail closed without crashing the predicate.
+    - Only tiers `None` / ``""`` / ``"standard"`` run anywhere. ANY other
+      tier value (an unknown or mistyped literal like ``"Sandboxed"``) is
+      treated like ``"sandboxed"`` and requires capability — no silent
+      downgrade to unsandboxed placement.
+
+    A second, independent gate applies to tasks carrying an `argv` payload
+    (arbitrary user command lines): the claiming node must advertise
+    `argv_capable is True`. This is checked BEFORE the isolation block's
+    `allowFallback` waiver below, and the waiver does not apply to it — a
+    submitter setting `allowFallback: true` waives the sandbox-tier
+    requirement only, never the argv-runner requirement, or arbitrary argv
+    could land on a node with no argv runner at all.
+
+    A third gate applies to tasks carrying a `module` payload (the
+    "python -m <allowlisted module>" tier): the claiming node must not be
+    `module_capable is False`. Its polarity is deliberately the MIRROR of
+    the argv gate above, not a copy of it:
+
+    - `argv_capable` is fail-closed (`is True` required) because argv is a
+      *safety* boundary — placing argv on a node with no argv runner is a
+      security failure (ArgvDockerRunner's own payload check would refuse
+      it, but the whole point of placement is to not burn attempts getting
+      there).
+    - `module_capable` is fail-OPEN (`is False` excludes) because it is an
+      *availability* concern only — a module task misplaced on an
+      argv-only node wastes retry attempts, it never escapes a sandbox.
+      Defaulting to capable means an already-deployed node whose
+      registration predates this field (module_capable absent ⇒ None) —
+      or any node that simply never opted into an argv-only runner — keeps
+      receiving module work. Only a node that explicitly advertises
+      `module_capable: false` (an argv-only volunteer) is excluded.
+
+    Do NOT "harmonize" these two gates onto the same polarity — copying
+    the argv gate's `is True` pattern onto module_capable would silently
+    cut every already-deployed node off from module work the moment this
+    field shipped, without anyone having opted out of anything.
+
+    Everything genuinely standard keeps the fail-open placement default."""
+
+    def eligible(self, task: TaskSpec, node: NodeView) -> bool:
+        # Checked before the allowFallback waiver below: the waiver relaxes
+        # the sandbox-tier requirement, and must never be readable as
+        # permission to run argv on a node with no argv runner.
+        if "argv" in task.payload and node.get("argv_capable") is not True:
+            return False
+        # Availability gate, mirrored polarity from the argv gate above —
+        # see the class docstring. An argv-only volunteer poisons every
+        # module job in the pool otherwise: it claims, ArgvDockerRunner
+        # rejects the payload, the attempt fails, and the task requeues
+        # into the same node's path until attempts are exhausted.
+        if "module" in task.payload and node.get("module_capable") is False:
+            return False
+        isolation = task.payload.get("isolation")
+        if isolation is None:
+            return True  # no isolation payload ⇒ standard, runs anywhere
+        if not isinstance(isolation, dict):
+            return False  # type-confused payload ⇒ fail closed, no crash
+        if isolation.get("tier") in (None, "", "standard"):
+            return True  # only the known non-isolated tiers run anywhere
+        if isolation.get("allowFallback") is True:
+            return True  # explicit waiver — genuine boolean only
+        return node.get("sandbox_capable") is True  # capable ⇒ genuine boolean only

flashruntime/sdk.py

@@ -0,0 +1,402 @@
+"""flash.submit(): run a CommandWorkload locally and hand back a Run.
+
+The execution model is compile → launch → wait → collect, once per Mode A
+param set (mirroring the M0 engine's replay model). Sequential-by-design
+keeps collection correct (a trial's outputs are copied out of the source
+dir before the next trial can overwrite them). Service submission is a
+different door: `workloads.command.to_jobspec()` POSTed to the coordinator.
+
+`wait=True` (default) drives that loop inline and returns a finished Run —
+byte-for-byte the old synchronous behavior. `wait=False` returns the Run
+immediately and drives the loop on a daemon thread, so the caller can watch
+it live. Either way the Run mirrors itself to `<output_dir>/run.json` after
+every state change (the `viewer_v1` contract Task 6 renders from disk).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import sys
+import tempfile
+import threading
+import time
+import webbrowser
+from pathlib import Path
+
+from flashruntime.launchers import LaunchState
+from flashruntime.launchers.local import LocalProcessLauncher
+from flashruntime.monitor import ResourceSampler
+from flashruntime.protocol.v1alpha1 import RecoveryActionType
+from flashruntime.recovery import classify, decide
+from flashruntime.recovery.signals import from_local_launch
+from flashruntime.strategies.command import compile_workload
+from flashruntime.workloads.command import CommandWorkload
+
+_JOB_ID = "local"  # deterministic: rerunning with the same output_dir resumes checkpoints
+
+# Viewers opened by watch=True. A RunViewerServer runs on a daemon thread, so
+# it never blocks interpreter exit; we hold a reference here so it survives for
+# the whole process lifetime (a wait=True submit returns before the human is
+# done looking at the page, and the server must outlive that return).
+_VIEWERS: list = []
+
+
+class Run:
+    """Result handle for one submit().
+
+    Lock discipline (stated once, here): a `wait=False` submit drives the
+    launch loop on a daemon thread while the caller reads state/events live.
+    A single `threading.Lock` (`_lock`) guards the live-read fields
+    (`state`, `finished_at`, `_events`, `_attempts`) *and* the run.json
+    write, so the file is always serialized from a consistent snapshot and
+    a concurrent reader never observes a half-updated Run. `trials`,
+    `artifacts`, and `_logs` are mutated only by the single driver thread
+    and serialized only under the lock — they need no lock of their own.
+    Readers take copies under the lock; the driver holds it only for the
+    brief mutate-then-persist critical section. `_done` (a `threading.Event`)
+    is set exactly once, on the terminal transition, so `wait()` blocks
+    without polling.
+    """
+
+    def __init__(self, workload: CommandWorkload, output_dir: Path, max_restarts: int = 0):
+        self.workload = workload
+        self.output_dir = Path(output_dir)
+        self.max_restarts = max_restarts  # the real recovery budget the viewer reports
+        self.state: LaunchState = LaunchState.PENDING
+        self.trials: list[dict] = []
+        self.artifacts: list[Path] = []
+        self.viewer_url: str | None = None  # set when submit(watch=True) opens a live viewer
+        self.started_at: float = time.time()
+        self.finished_at: float | None = None
+        self._logs: list[str] = []
+        self._events: list[dict] = []
+        self._attempts: list[dict] = []
+        self._lock = threading.Lock()
+        self._done = threading.Event()
+        with self._lock:  # materialize the PENDING record so run.json exists at once
+            self._write_run_json()
+
+    @property
+    def run_json_path(self) -> Path:
+        """Path to the viewer_v1 document this Run mirrors itself to."""
+        return self.output_dir / "run.json"
+
+    @property
+    def events(self) -> list[dict]:
+        """Snapshot copy of the append-only event log — safe to read while
+        the driver thread appends (a bare reference could tear)."""
+        with self._lock:
+            return [dict(e) for e in self._events]
+
+    @property
+    def attempts(self) -> list[dict]:
+        """Snapshot copy of the per-launch attempt rows (see `events` for why
+        it copies)."""
+        with self._lock:
+            return [dict(a) for a in self._attempts]
+
+    def wait(self, timeout: float | None = None) -> LaunchState:
+        """Block until the run is terminal (or `timeout` seconds elapse),
+        then return the current state. Event-based, not a poll loop: the
+        driver sets `_done` on the terminal transition, so a waiter wakes the
+        instant the run finishes with zero CPU spin — a polling loop would
+        burn cycles and add up-to-interval latency. Callers check
+        `.terminal` on the result (a timeout returns the last state seen)."""
+        self._done.wait(timeout)
+        return self.state
+
+    def record_event(self, type: str, message: str) -> None:
+        """Append one event to the append-only log and re-persist run.json.
+
+        Reason: lifecycle/recovery decisions must reach the viewer (and Task
+        4's recovery wiring) the moment they happen. Takes `_lock` so the
+        append and the file write are one critical section."""
+        with self._lock:
+            self._events.append({"ts": time.time(), "type": type, "message": message})
+            self._write_run_json()
+
+    def _set_state(self, state: LaunchState) -> None:
+        """Transition to `state` and re-persist run.json. On a terminal
+        state, stamp `finished_at` and release `wait()`ers (sets `_done`).
+        Under `_lock` so state + finished_at + write land atomically."""
+        with self._lock:
+            self.state = state
+            if state.terminal and self.finished_at is None:
+                self.finished_at = time.time()
+            self._write_run_json()
+            if state.terminal:
+                self._done.set()
+
+    def _add_attempt(self, attempt_id: str, job_id: str, handle, started_at: float) -> None:
+        """Append a RUNNING attempt row (one per launch) and persist. `pid`
+        is the launcher's execution_id (a string), so the viewer can show
+        what is live."""
+        with self._lock:
+            self._attempts.append(
+                {
+                    "attempt_id": attempt_id,
+                    "job_id": job_id,
+                    "state": LaunchState.RUNNING.value,
+                    "pid": handle.execution_id,
+                    "started_at": started_at,
+                    "finished_at": None,
+                    "output_dir": str(handle.output_dir),
+                }
+            )
+            self._write_run_json()
+
+    def _finish_attempt(self, attempt_id: str, state: LaunchState) -> None:
+        """Settle an attempt's terminal state + finished_at and persist.
+        Matches the last row with this id, so restarts (Task 4) that append a
+        fresh row settle the right one."""
+        with self._lock:
+            for row in reversed(self._attempts):
+                if row["attempt_id"] == attempt_id:
+                    row["state"] = state.value
+                    row["finished_at"] = time.time()
+                    break
+            self._write_run_json()
+
+    def _write_run_json(self) -> None:
+        """Serialize the Run to <output_dir>/run.json atomically.
+
+        Write a temp file in the same directory, then `os.replace` it onto
+        the final path: os.replace is atomic on POSIX and Windows, so a
+        viewer reading concurrently sees either the whole old file or the
+        whole new one — never a torn, half-written document. Callers hold
+        `_lock`, so the snapshot serialized here is internally consistent."""
+        doc = {
+            "contract": "viewer_v1",
+            "workload": {
+                "command": self.workload.argv(),
+                "mode": self.workload.resolved_mode(),
+                "source": self.workload.source.path,
+            },
+            "state": self.state.value,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "max_restarts": self.max_restarts,
+            "attempts": [dict(a) for a in self._attempts],
+            "events": [dict(e) for e in self._events],
+            "trials": [dict(t) for t in self.trials],
+        }
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        tmp = self.output_dir / f".run.json.{os.getpid()}.tmp"
+        tmp.write_text(json.dumps(doc, indent=2))
+        os.replace(tmp, self.run_json_path)
+
+    def logs(self, tail_lines: int = 200) -> str:
+        return "\n".join("\n".join(self._logs).splitlines()[-tail_lines:])
+
+    def best_trial(self, metric: str | None = None, maximize: bool | None = None) -> dict | None:
+        """Highest/lowest `metric` among trials that reported it. Defaults
+        come from the workload's OutputSpec (adapters set them)."""
+        metric = metric or self.workload.outputs.primary_metric
+        if maximize is None:
+            maximize = self.workload.outputs.maximize
+        if metric is None:
+            raise ValueError("no metric named: pass metric= or set outputs.primary_metric")
+        scored = [t for t in self.trials if metric in t]
+        if not scored:
+            return None
+        return max(scored, key=lambda t: t[metric]) if maximize else min(scored, key=lambda t: t[metric])
+
+
+def _watch_enabled(watch: bool | None) -> bool:
+    """Resolve the `watch` tri-state. Explicit True/False wins; None is auto —
+    on only at an interactive terminal AND outside CI. A pipeline or a piped
+    stdout must never open a browser or hold a server open, so auto stays off
+    there even though the run is identical."""
+    if watch is not None:
+        return watch
+    return sys.stdout.isatty() and os.environ.get("CI") is None
+
+
+def _open_viewer(run: Run) -> None:
+    """Start a read-only RunViewerServer over the run's output dir, record its
+    URL on the Run, open a browser at it, and keep the server alive for the
+    process lifetime (`_VIEWERS`). Best-effort by design: the viewer is a
+    convenience layered on top of the run, never load-bearing — a headless box
+    with no browser, or a port that will not bind, must still run the job. So
+    every failure here is swallowed and the submit proceeds unwatched."""
+    try:
+        from flashruntime.viewer.server import RunViewerServer
+
+        server = RunViewerServer(run.output_dir)
+        url = server.start()
+        _VIEWERS.append(server)  # hold a reference: daemon thread, freed at exit
+        run.viewer_url = url
+        print(f"viewer: {url}")
+        try:
+            webbrowser.open(url)
+        except Exception:  # noqa: BLE001 — headless / no browser must not fail the run
+            pass
+    except Exception:  # noqa: BLE001 — the viewer must never be able to fail a submit
+        pass
+
+
+def submit(
+    workload: CommandWorkload,
+    output_dir: str | Path | None = None,
+    wait: bool = True,
+    max_restarts: int = 0,
+    watch: bool | None = None,
+) -> Run:
+    """Run a CommandWorkload locally. `max_restarts` (default 0 — no retry, the
+    old behavior) is the automatic fault-tolerance budget: a FAILED attempt is
+    classified and run against the versioned recovery policy, and unless the
+    failure is a deterministic application error (FAIL_JOB → fail fast) the
+    same spec is relaunched from the job-scoped checkpoint, up to this many
+    times. Applies to a single launch and per-trial in a fan-out.
+
+    `watch` opens the live run page (`flashruntime.viewer`) in a browser and
+    prints its URL (recorded on `Run.viewer_url`). None (default) auto-decides:
+    on at an interactive terminal, off in pipes/CI. The viewer server binds
+    127.0.0.1 on a daemon thread, so it works for both `wait=True` (it outlives
+    the synchronous return, so the page stays viewable) and `wait=False`."""
+    out_root = Path(output_dir) if output_dir else Path(tempfile.mkdtemp(prefix="flashruntime-run-"))
+    run = Run(workload, out_root, max_restarts=max_restarts)
+    launcher = LocalProcessLauncher(out_root)
+
+    # Open the viewer before driving so the page is live for the whole run —
+    # including a wait=False submit the caller watches from the first frame.
+    if _watch_enabled(watch):
+        _open_viewer(run)
+
+    fanout = workload.resolved_mode() == "independent_tasks" and workload.task_params
+    param_sets: list[dict | None] = list(workload.task_params) if fanout else [None]
+
+    if wait:
+        # inline drive == today's synchronous behavior (Run finished on return)
+        _drive(run, workload, launcher, param_sets)
+    else:
+        # daemon so an unwaited Run never blocks interpreter shutdown
+        threading.Thread(
+            target=_drive, args=(run, workload, launcher, param_sets), daemon=True
+        ).start()
+    return run
+
+
+def _drive(
+    run: Run, workload: CommandWorkload, launcher: LocalProcessLauncher, param_sets: list
+) -> None:
+    """The compile → launch → wait → collect loop, once per param set, then
+    settle the Run's terminal state. Extracted from submit() so wait=True can
+    call it inline (identical to the old synchronous path) and wait=False can
+    run it on a daemon thread.
+
+    Around each param set is the automatic-recovery loop (Task 4): on a FAILED
+    launch, translate the exit into FailureSignals, classify it, and run the
+    versioned policy — fail fast on a deterministic bug (FAIL_JOB), else
+    relaunch from the job-scoped checkpoint until the restart budget is spent.
+    `run.max_restarts == 0` reduces this to exactly one launch, the pre-Task-4
+    behavior."""
+    run._set_state(LaunchState.RUNNING)
+    source_dir = Path(workload.source.path).expanduser()
+
+    fanout = workload.resolved_mode() == "independent_tasks" and workload.task_params
+    # The policy's blast-radius axis: independent trials retry one task; a
+    # single/coordinated run restarts the whole (coordinated-training) group.
+    mode = "independent_tasks" if fanout else "coordinated_training"
+
+    states: list[LaunchState] = []
+    for i, params in enumerate(param_sets):
+        base_attempt = f"task-{i:03d}"
+        # Fan-out trials are DIFFERENT workloads (distinct params) — each needs
+        # its own checkpoint tree, or trial i could restore trial j's weights
+        # (FLASHML_CKPT_DIR is per job id). The single-workload path keeps the
+        # stable "local" id so a resubmit against the same output_dir resumes.
+        job_id = f"local-{i:03d}" if fanout else _JOB_ID
+        spec = compile_workload(workload, params)
+
+        final_state = LaunchState.FAILED
+        trial_metrics: dict | None = None
+        # restart 0 is the first try; 1..max_restarts are recovery attempts.
+        for restart in range(run.max_restarts + 1):
+            # Restart attempts append -r1/-r2… but keep the SAME job_id, so the
+            # launcher's job-scoped FLASHML_CKPT_DIR is shared across attempt
+            # names — that is the whole trick: the child's ft.prepare() resumes
+            # from the predecessor's newest valid manifest even though this
+            # attempt has a different id.
+            attempt_id = base_attempt if restart == 0 else f"{base_attempt}-r{restart}"
+            started_at = time.time()
+            handle = launcher.launch(spec, job_id, attempt_id)
+            run.record_event("LAUNCH_STARTED", f"{attempt_id} launched (pid {handle.execution_id})")
+            run._add_attempt(attempt_id, job_id, handle, started_at)
+            # Telemetry beside the attempt (machine + process tree → the run
+            # viewer's flow map). Best-effort like the viewer itself: a
+            # sampler that cannot start must never fail the run.
+            sampler = None
+            try:
+                sampler = ResourceSampler(handle.output_dir, int(handle.execution_id))
+                sampler.start()
+            except Exception:  # noqa: BLE001 — observability never fails a run
+                sampler = None
+            try:
+                final_state = handle.wait()
+            finally:
+                if sampler is not None:
+                    sampler.stop()
+            run._logs.append(f"--- {attempt_id} ({final_state.value}) ---\n{handle.logs()}")
+            run._finish_attempt(attempt_id, final_state)
+
+            collected = _collect(source_dir, workload.outputs.collect, handle.output_dir, since=started_at)
+            run.artifacts.extend(collected)
+            metrics_path = handle.output_dir / "metrics.json"
+            if metrics_path.is_file():
+                try:
+                    metrics = json.loads(metrics_path.read_text())
+                except ValueError:
+                    metrics = None
+                if isinstance(metrics, dict):
+                    if params:
+                        metrics.setdefault("params", params)
+                    trial_metrics = metrics  # the winning attempt's metrics (one trial per param set)
+
+            if final_state is LaunchState.SUCCEEDED:
+                break
+
+            # FAILED: consult the recovery policy. Signals → class → decision,
+            # both events recorded (with the decision's human reason) so the
+            # ledger/viewer can explain every retry.
+            exit_code = handle.exit_code
+            decision = decide(classify(from_local_launch(exit_code, handle.logs())), mode)
+            run.record_event(
+                "FAILURE_CLASSIFIED", f"{attempt_id}: {decision.failure_class.value} (exit {exit_code})"
+            )
+            run.record_event(
+                "RECOVERY_ACTION_SELECTED", f"{attempt_id}: {decision.action.value} — {decision.reason}"
+            )
+            # Fail fast on a deterministic application error — a retry only
+            # re-hits the same bug. Any other action means "retry", and the
+            # loop's range() enforces the budget, so it cannot spin forever.
+            if decision.action is RecoveryActionType.FAIL_JOB:
+                break
+
+        if trial_metrics is not None:
+            run.trials.append(trial_metrics)
+        states.append(final_state)
+
+    run._set_state(
+        LaunchState.SUCCEEDED
+        if states and all(s is LaunchState.SUCCEEDED for s in states)
+        else LaunchState.FAILED
+    )
+
+
+def _collect(source_dir: Path, patterns: list[str], dest: Path, since: float) -> list[Path]:
+    """Copy collect-globs from the script's cwd into the attempt's output
+    dir. `since` skips files older than this launch — a stale metrics.json
+    from a previous trial must never be credited to a failed one."""
+    out: list[Path] = []
+    for pattern in patterns:
+        for src in sorted(source_dir.glob(pattern)):
+            if not src.is_file() or src.stat().st_mtime < since:
+                continue
+            target = dest / src.relative_to(source_dir)
+            target.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(src, target)
+            out.append(target)
+    return out

flashruntime/service/__init__.py

@@ -0,0 +1,3 @@
+"""FlashRuntime API service: accepts JobSpecs, drives an ExecutionBackend,
+keeps the job/event/artifact ledger, and serves it to FlashML Cloud (or any
+self-hosted client — the runtime is useful without the cloud)."""