flashruntime 0.3.0__py3-none-any.whl

flashruntime/recipes/__init__.py

@@ -0,0 +1,135 @@
+"""Workload recipes: everything one workload type needs, behind one interface.
+
+Today each workload is wired by hand in three places: an expansion branch
+in `service/modea.py`, a module allowlist entry (both ends), and a task
+module in `flashml_workloads/`. A `WorkloadRecipe` bundles those into one
+registrable object so adding a workload becomes: write the task module,
+write the recipe, register it. The existing hand-rolled expansions
+(`hyperparameter_search`, `sharded_kmeans`) are the reference behavior and
+should migrate here without any wire-visible change.
+
+A recipe spans three moments in a task's life:
+  submit-time  — `validate_params` + `expand` (coordinator)
+  run-time     — `task_module` executed by the agent (the §2.2 contract:
+                 `python -m <module> --spec spec.json --out OUTDIR`)
+  commit-time  — `validate_output` (coordinator; runs BEFORE acceptance,
+                 alongside the sha256 check) and, for map/reduce shapes,
+                 `reduce` (driver side)
+
+Hard rules inherited from the system: the task module must appear on BOTH
+allowlists (coordinator + agent) — a recipe self-declares it, and the
+registry is what the allowlists will be generated from once migration
+lands. Hugging Face code lives here and only here (four-axes rule): HF is
+workload, never a backend.
+
+Status: interface complete (final surface); first concrete recipe is
+HF Trainer + PEFT LoRA (SPRINT_PLAN Days 8–10, design in ADR-0004-to-be).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, ClassVar
+
+from flashruntime.protocol.v1alpha1 import JobSpec, TaskSpec
+
+__all__ = ["WorkloadRecipe", "register_recipe", "recipe_for"]
+
+
+class WorkloadRecipe(ABC):
+    """One workload type, end to end.
+
+    Implementations must be *pure coordination logic*: no framework
+    imports at module import time (import torch/transformers inside the
+    task module, which runs on the agent — never in the coordinator's
+    process).
+    """
+
+    #: The `JobSpec.spec.workload.type` value this recipe owns
+    #: (e.g. "hyperparameter_search", "sharded_kmeans", "lora_finetune").
+    kind: ClassVar[str]
+
+    #: The task module executed on agents — the value placed in every
+    #: payload's "module" field, and the entry both allowlists must carry.
+    task_module: ClassVar[str]
+
+    def validate_params(self, params: dict[str, Any]) -> list[str]:
+        """Submit-time parameter validation.
+
+        Returns every problem found (empty list = valid); the service
+        turns a non-empty list into one 422 carrying all reasons — the
+        planner's surface-everything-at-once UX, applied to submission.
+        Default: accept anything (expand() remains the backstop).
+        """
+        return []
+
+    @abstractmethod
+    def expand(self, job_id: str, spec: JobSpec) -> list[TaskSpec]:
+        """Turn a JobSpec into independent TaskSpecs (Mode A expansion).
+
+        Requirements (mirror the existing expansions exactly):
+        - deterministic task_ids (`trial-000`-style) — retries and
+          dashboards depend on stable names;
+        - every payload carries: module, params, inputs (artifact:// URIs),
+          output_prefix, task_id, image, and `checkpoint` when the recipe
+          is checkpointable;
+        - commit_key = the metrics.json path under output_prefix (the
+          idempotency + validation anchor);
+        - honor spec.spec.retryPolicy.maxTaskAttempts and the workload's
+          lease_seconds parameter.
+        Raises ValueError (→ 422) when the spec cannot be expanded.
+        """
+
+    @abstractmethod
+    def validate_output(self, metrics: dict[str, Any]) -> None:
+        """Commit-time semantic validation of a task's metrics.json.
+
+        Runs on the coordinator after the sha256 check passes and before
+        the commit is accepted: raise (ValueError with a reason) to fail
+        the attempt — the task requeues, exactly like a hash mismatch.
+        This is the hook that catches \"the file is intact but the result
+        is nonsense\" (NaN loss, missing fields, accuracy > 1.0).
+        """
+
+    def reduce(self, outputs: list[dict[str, Any]]) -> dict[str, Any] | None:
+        """Combine per-task outputs into a job-level result (map/reduce
+        workloads: K-means partials → new centroids; HPO trials → best
+        config). Return None for workloads with no reduce step (default).
+        Runs driver/coordinator-side on already-validated outputs."""
+        return None
+
+    def checkpointable(self) -> bool:
+        """Whether tasks of this recipe write resumable checkpoints (turns
+        on the agent's checkpoint relay via the payload's `checkpoint`
+        key). Default False: stateless tasks retry from scratch, which is
+        cheaper than checkpointing for short work."""
+        return False
+
+    def plan_hints(self) -> dict[str, Any]:
+        """Optional coupling to the planner: static facts the estimators
+        may use (e.g. {'mode': 'independent_tasks'}). Keep it data-only —
+        the planner never calls recipe *code* (determinism + no-framework
+        rules)."""
+        return {}
+
+
+_REGISTRY: dict[str, WorkloadRecipe] = {}
+
+
+def register_recipe(recipe: WorkloadRecipe) -> None:
+    """Register (or replace) the recipe for its `kind`. At service startup
+    the registered kinds become the expansion dispatch table, and the set
+    of `task_module`s becomes the coordinator-side allowlist — one source
+    of truth instead of today's two hand-maintained sets."""
+    _REGISTRY[recipe.kind] = recipe
+
+
+def recipe_for(kind: str) -> WorkloadRecipe:
+    """Lookup by workload type; LookupError → the service's 422 listing
+    the supported kinds (same UX as the current ExpansionError)."""
+    try:
+        return _REGISTRY[kind]
+    except KeyError:
+        raise LookupError(
+            f"no recipe for workload type {kind!r} (registered: {sorted(_REGISTRY) or 'none'})"
+        )

flashruntime/recipes/command.py

@@ -0,0 +1,166 @@
+"""The generic command recipe: JobSpec{workload.type: "command"} → lease tasks.
+
+The first concrete WorkloadRecipe. Payloads carry `argv` — the §2.2
+executor contract generalized from `module` — plus the isolation
+requirement the placement gate enforces fail-closed. Executing argv
+payloads is flashnode's runner tier (cross-repo, versioned change); this
+recipe defines the coordinator half of that contract. Until flashnode
+ships it, command jobs expand and lease correctly but only argv-aware
+executors can run them.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, ClassVar
+
+from flashruntime.protocol.v1alpha1 import JobSpec, TaskSpec
+from flashruntime.recipes import WorkloadRecipe, register_recipe
+
+
+class CommandRecipe(WorkloadRecipe):
+    kind: ClassVar[str] = "command"
+    #: argv payloads name no task module — the isolation tier, not a module
+    #: allowlist, is the security control for this workload type.
+    task_module: ClassVar[str] = ""
+
+    def validate_params(self, params: dict[str, Any]) -> list[str]:
+        problems: list[str] = []
+        command = params.get("command")
+        if (
+            not command
+            or not isinstance(command, list)
+            or not all(isinstance(t, str) for t in command)
+        ):
+            problems.append("'command' must be a non-empty argv list of strings")
+        for name, uri in (params.get("inputs") or {}).items():
+            if not str(uri).startswith("artifact://"):
+                problems.append(f"input '{name}' must be an artifact:// URI")
+        task_params = params.get("task_params")
+        if task_params is not None and (
+            not isinstance(task_params, list)
+            or not all(isinstance(p, dict) for p in task_params)
+        ):
+            problems.append("'task_params' must be a list of objects")
+        problems.extend(self._unpack_problems(params))
+        return problems
+
+    @staticmethod
+    def _unpack_problems(params: dict[str, Any]) -> list[str]:
+        """Validate `unpack_inputs` — the names flashnode may run an archive
+        extractor over.
+
+        This value decides which downloaded bytes get unpacked into a
+        directory tree on a *volunteer's* machine, so it is validated at
+        expansion time rather than forwarded as given. Naming an input that
+        was never declared is the interesting case: flashnode refuses it
+        too, but only after the task has been leased, claimed and half-run,
+        which surfaces to the submitter as a mysterious node-side failure
+        instead of the spec error it is. Catching it here fails on the
+        submitter's side, before anything is placed.
+        """
+        unpack = params.get("unpack_inputs")
+        if unpack is None:
+            return []
+        if not isinstance(unpack, list) or not all(isinstance(n, str) for n in unpack):
+            return ["'unpack_inputs' must be a list of input names (strings)"]
+        problems: list[str] = []
+        duplicates = sorted({n for n in unpack if unpack.count(n) > 1})
+        if duplicates:
+            problems.append(
+                f"'unpack_inputs' names {duplicates} more than once — an input "
+                f"is unpacked at most once"
+            )
+        declared = set((params.get("inputs") or {}).keys())
+        unknown = sorted(set(unpack) - declared)
+        if unknown:
+            problems.append(
+                f"'unpack_inputs' names inputs that are not declared in "
+                f"'inputs': {unknown}"
+            )
+        return problems
+
+    def expand(self, job_id: str, spec: JobSpec) -> list[TaskSpec]:
+        isolation_spec = spec.spec.isolation
+        if isolation_spec.allowFallback:
+            # allowFallback waives the sandbox capability requirement at
+            # placement time. Honouring it for argv would let a submitter
+            # place arbitrary code on an unsandboxed node.
+            raise ValueError(
+                "command jobs may not set isolation.allowFallback — "
+                "argv execution is container-only"
+            )
+        if isolation_spec.tier != "sandboxed":
+            # Coordinator-side opt-in only: the operator running the pool
+            # decides, never the submitter.
+            if os.environ.get("FLASHML_ALLOW_UNSANDBOXED_ARGV") != "1":
+                raise ValueError(
+                    f"command jobs require isolation.tier 'sandboxed', got "
+                    f"{isolation_spec.tier!r} (set FLASHML_ALLOW_UNSANDBOXED_ARGV=1 "
+                    f"on the coordinator to allow a trusted fleet)"
+                )
+        p = spec.spec.workload.parameters
+        problems = self.validate_params(p)
+        if problems:
+            raise ValueError("; ".join(problems))
+
+        param_sets: list[dict | None] = p.get("task_params") or [None]
+        env: dict[str, str] = dict(p.get("env") or {})
+        inputs = dict(p.get("inputs") or {})
+        isolation = {
+            "tier": spec.spec.isolation.tier,
+            "allowFallback": spec.spec.isolation.allowFallback,
+        }
+
+        tasks: list[TaskSpec] = []
+        for i, params in enumerate(param_sets):
+            task_id = f"task-{i:03d}"
+            try:
+                argv = [t.format(**params) for t in p["command"]] if params else list(p["command"])
+                task_env = {
+                    k: (v.format(**params) if params else v) for k, v in env.items()
+                }
+            except (KeyError, IndexError, ValueError) as exc:
+                # KeyError: a named {placeholder} with no matching param.
+                # IndexError/ValueError: an auto/positional field ({}, {0}) or
+                # a malformed brace str.format cannot fill from a params dict.
+                # All are user-input errors → ValueError (→422), never a 500.
+                raise ValueError(
+                    f"task {i}: placeholder {exc} has no value in task_params[{i}]"
+                ) from None
+            payload: dict[str, Any] = {
+                "argv": argv,
+                "env": task_env,
+                "inputs": inputs,
+                "output_prefix": f"jobs/{job_id}/{task_id}/",
+                "task_id": task_id,
+                "image": spec.spec.image.reference,
+                "isolation": isolation,
+            }
+            if p.get("checkpoint") is not None:
+                payload["checkpoint"] = p["checkpoint"]
+            if p.get("unpack_inputs") is not None:
+                # Absent stays absent, never an empty list: flashnode reads
+                # `payload.get("unpack_inputs")` and an omitted key is the
+                # path where every input keeps its plain-file behaviour byte
+                # for byte. Emitting `[]` would mean the same thing today but
+                # would stop exercising that path.
+                payload["unpack_inputs"] = list(p["unpack_inputs"])
+            tasks.append(
+                TaskSpec(
+                    task_id=task_id,
+                    job_id=job_id,
+                    commit_key=f"jobs/{job_id}/{task_id}/metrics.json",
+                    max_attempts=spec.spec.retryPolicy.maxTaskAttempts,
+                    lease_seconds=float(p.get("lease_seconds", 60.0)),
+                    payload=payload,
+                )
+            )
+        return tasks
+
+    def validate_output(self, metrics: dict[str, Any]) -> None:
+        if not isinstance(metrics, dict):
+            raise ValueError("metrics.json must contain a JSON object")
+
+
+register_recipe(CommandRecipe())

flashruntime/recovery/__init__.py

@@ -0,0 +1,21 @@
+"""Failure taxonomy and the deterministic recovery policy.
+
+Two pure functions and a versioned table:
+
+    from flashruntime.recovery import FailureSignals, classify, decide
+
+    failure = classify(FailureSignals(heartbeat_lost=True))   # NODE_LOSS
+    decision = decide(failure, mode="independent_tasks")
+    # → RETRY_TASK, cordon_node=True, policy_version=0.1.0, reason=...
+
+`classify` turns raw observed signals (exit codes, heartbeat loss, NCCL
+messages, XID events) into one typed FailureClass with precedence-ordered
+rules. `decide` maps (FailureClass, execution mode) to one typed, logged
+RecoveryDecision. No scoring, no learning, no agent: same failure + same
+policy version ⇒ same action.
+"""
+
+from flashruntime.recovery.policy import POLICY_VERSION, decide
+from flashruntime.recovery.taxonomy import CORRELATED_THRESHOLD, FailureSignals, classify
+
+__all__ = ["FailureSignals", "classify", "decide", "POLICY_VERSION", "CORRELATED_THRESHOLD"]

flashruntime/recovery/policy.py

@@ -0,0 +1,170 @@
+"""The versioned, deterministic recovery policy.
+
+FailureClass × execution mode → one typed `RecoveryDecision`. The table *is*
+the policy: no scoring, no learning, no agent — same failure + same policy
+version ⇒ same action, always (ADR-0003 / master report guardrail: recovery
+actions are typed, authorized, and logged).
+
+The two execution modes matter because the same failure has different blast
+radii: a worker crash in Mode A costs one task retry; in Mode B it stops the
+whole group (NCCL state is not repairable in place — whole-group restart
+from the latest valid checkpoint is the honest v1 promise).
+
+Encoded rules worth naming:
+- Deterministic application errors are never retried — fail fast, tell the
+  user. Burning capacity on a bug is the most expensive kind of "recovery".
+- Storage outages pause rather than kill: compute waiting on dead storage
+  is wasted, but the job's state is intact — protect it and stop the burn.
+- Correlated incidents freeze automation entirely. Retry storms during a
+  systemic incident are how orchestrators destroy trust; the policy's most
+  important action is knowing when to stop acting.
+"""
+
+from __future__ import annotations
+
+from flashruntime.protocol.v1alpha1 import (
+    FailureClass,
+    RecoveryActionType,
+    RecoveryDecision,
+)
+
+POLICY_VERSION = "0.1.0"
+
+Mode = str  # "independent_tasks" | "coordinated_training"
+
+# (action, scope, cordon_node, needs_checkpoint, reason)
+_TABLE: dict[tuple[FailureClass, Mode], tuple[RecoveryActionType, str, bool, bool, str]] = {
+    # -- application / data: not infrastructure's fault ---------------------
+    (FailureClass.APPLICATION_ERROR, "independent_tasks"): (
+        RecoveryActionType.FAIL_JOB, "job", False, False,
+        "deterministic application error — retrying burns money on a bug",
+    ),
+    (FailureClass.APPLICATION_ERROR, "coordinated_training"): (
+        RecoveryActionType.FAIL_JOB, "job", False, False,
+        "deterministic application error — retrying burns money on a bug",
+    ),
+    (FailureClass.DATA_ERROR, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "data error on one shard — retry once elsewhere; quarantine on repeat",
+    ),
+    (FailureClass.DATA_ERROR, "coordinated_training"): (
+        RecoveryActionType.FAIL_JOB, "job", False, False,
+        "data error in coordinated training — resume would repeat it",
+    ),
+    # -- process / node -----------------------------------------------------
+    (FailureClass.WORKER_CRASH, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "non-deterministic worker crash — lease expires, task requeues",
+    ),
+    (FailureClass.WORKER_CRASH, "coordinated_training"): (
+        RecoveryActionType.RESTART_GROUP, "group", False, True,
+        "a lost rank stops the group — restart all workers from latest valid checkpoint",
+    ),
+    (FailureClass.NODE_LOSS, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", True, False,
+        "node gone — cordon it, expire its leases, requeue its tasks",
+    ),
+    (FailureClass.NODE_LOSS, "coordinated_training"): (
+        RecoveryActionType.REPLACE_NODE, "group", True, True,
+        "node gone — acquire replacement capacity, then group restart from checkpoint",
+    ),
+    (FailureClass.ACCELERATOR_FAILURE, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "node", True, False,
+        "GPU/driver fault — cordon the accelerator's node, retry elsewhere",
+    ),
+    (FailureClass.ACCELERATOR_FAILURE, "coordinated_training"): (
+        RecoveryActionType.REPLACE_NODE, "group", True, True,
+        "GPU/driver fault — replace the node, group restart from checkpoint",
+    ),
+    # -- communication / network -------------------------------------------
+    (FailureClass.COMMUNICATION_ERROR, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "transport error on an independent task — plain retry",
+    ),
+    (FailureClass.COMMUNICATION_ERROR, "coordinated_training"): (
+        RecoveryActionType.RESTART_GROUP, "group", False, True,
+        "NCCL-class error — collective state is not repairable in place; whole-group restart",
+    ),
+    (FailureClass.NETWORK_DEGRADATION, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "degraded link — move the task; the node's reliability score absorbs the signal",
+    ),
+    (FailureClass.NETWORK_DEGRADATION, "coordinated_training"): (
+        RecoveryActionType.RESTART_GROUP, "group", False, True,
+        "sustained degradation starves the collective — restart, prefer a better pool",
+    ),
+    # -- dependencies -------------------------------------------------------
+    (FailureClass.STORAGE_TIMEOUT, "independent_tasks"): (
+        RecoveryActionType.PAUSE_JOB, "job", False, False,
+        "storage outage — pause instead of burning compute against a dead dependency",
+    ),
+    (FailureClass.STORAGE_TIMEOUT, "coordinated_training"): (
+        RecoveryActionType.PAUSE_JOB, "job", False, False,
+        "storage outage — pause instead of burning compute against a dead dependency",
+    ),
+    (FailureClass.ARTIFACT_CORRUPTION, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "output failed validation — reject commit, retry on a different node, score down producer",
+    ),
+    (FailureClass.ARTIFACT_CORRUPTION, "coordinated_training"): (
+        RecoveryActionType.RESTART_GROUP, "group", False, True,
+        "corrupt checkpoint/artifact — restart from the previous *valid* manifest",
+    ),
+    (FailureClass.PREEMPTION, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "priced-in preemption — requeue; no penalty to anyone",
+    ),
+    (FailureClass.PREEMPTION, "coordinated_training"): (
+        RecoveryActionType.REPLACE_NODE, "group", False, True,
+        "spot capacity reclaimed — acquire replacement, restart from checkpoint",
+    ),
+    # -- systemic: stop acting ---------------------------------------------
+    (FailureClass.CORRELATED_INCIDENT, "independent_tasks"): (
+        RecoveryActionType.FREEZE_AUTOMATION, "pool", False, False,
+        "multiple simultaneous failures — no retry storms; preserve state and escalate",
+    ),
+    (FailureClass.CORRELATED_INCIDENT, "coordinated_training"): (
+        RecoveryActionType.FREEZE_AUTOMATION, "pool", False, False,
+        "multiple simultaneous failures — no retry storms; preserve state and escalate",
+    ),
+    (FailureClass.CONTROL_PLANE_FAILURE, "independent_tasks"): (
+        RecoveryActionType.PAUSE_JOB, "job", False, False,
+        "coordinator unreachable — workers finish current leases; no new grants",
+    ),
+    (FailureClass.CONTROL_PLANE_FAILURE, "coordinated_training"): (
+        RecoveryActionType.PAUSE_JOB, "job", False, False,
+        "coordinator unreachable — preserve state; no automated decisions without a ledger",
+    ),
+    (FailureClass.UNKNOWN, "independent_tasks"): (
+        RecoveryActionType.RETRY_TASK, "task", False, False,
+        "unclassified failure — one conservative retry; repeats escalate via attempt limits",
+    ),
+    (FailureClass.UNKNOWN, "coordinated_training"): (
+        RecoveryActionType.PAUSE_JOB, "job", False, True,
+        "unclassified failure in coordinated training — pause for inspection over blind restart",
+    ),
+}
+
+
+def decide(failure: FailureClass, mode: Mode, evidence: dict | None = None) -> RecoveryDecision:
+    """Look up the typed action for a classified failure in a given mode.
+
+    Pure table lookup — deliberately no other inputs. Anything that should
+    influence the action (repeat counts, correlation) must appear upstream
+    as a different FailureClass, keeping every decision explainable by
+    (policy_version, failure_class, mode) alone.
+    """
+    try:
+        action, scope, cordon, needs_ckpt, reason = _TABLE[(failure, mode)]
+    except KeyError as exc:
+        raise ValueError(f"no policy entry for {failure.value} in mode {mode!r}") from exc
+    return RecoveryDecision(
+        policy_version=POLICY_VERSION,
+        failure_class=failure,
+        action=action,
+        scope=scope,  # type: ignore[arg-type]
+        cordon_node=cordon,
+        needs_checkpoint=needs_ckpt,
+        reason=reason,
+        evidence=evidence or {},
+    )

flashruntime/recovery/signals.py

@@ -0,0 +1,135 @@
+"""Translate a finished local process into recovery `FailureSignals`.
+
+`from_local_launch(exit_code, log_tail)` is the first real caller of the
+recovery package: it turns the only evidence a `LocalProcessLauncher` can
+observe — the child's OS exit code and the tail of its captured stdout+stderr
+— into the typed `FailureSignals` that `classify()` reads. It is a transparent
+lookup table, not an inference engine: every rule is one `if` whose comment
+names the reason the pattern implies its class, and the *first* matching rule
+wins (order encodes priority — deterministic-bug evidence outranks the
+transient default).
+
+Rule table (checked top-to-bottom, first match wins). One rule, one reason:
+
+  1. exit 0 / never started        → neutral signals (nothing broke; a guard
+                                      that keeps the function total).
+  2. a named deterministic
+     exception on a traceback
+     TERMINAL line                 → APPLICATION-ERROR (`exit_deterministic`):
+                                      import/parse/name errors recur
+                                      byte-for-byte, so fail fast, don't pay
+                                      for a re-run to the same certainty.
+  3. a bare "Traceback" that is
+     NOT the torchrun wrapper       → APPLICATION-ERROR: an unhandled exception
+                                      is a code error by default.
+  4. everything else (signal death
+     / OOM / bare SystemExit /
+     torchrun ChildFailedError)    → WORKER-CRASH (the transient default):
+                                      bad luck, worth one fresh attempt.
+
+Why rule 2 is anchored to a traceback TERMINAL line, not a bare substring:
+CPython prints the failing exception type at the START of the traceback's
+final line — `ModuleNotFoundError: ...`, or dotted with its defining module,
+`pkg.mod.SomeError: ...` — whereas prose only ever *mentions* an exception
+name mid-line. The canonical false positive is the startup warning
+`ImportError: flash_attn not available, falling back to eager attention`,
+which real loggers emit behind a timestamp/level prefix
+(`2026-… WARNING ImportError: …`) so the name never begins the line. A plain
+`substring in log` scan read that prose as a deterministic bug and failed the
+job fast — and because fail-fast is terminal, that was the worst-direction
+false positive: a transient crash that would have resumed instead never
+retried. Line-anchoring keeps every genuine traceback terminal (rule 2 still
+fires on the real thing) while ignoring the name wherever it appears in prose.
+
+Interaction of rules 2, 3 and the torchrun carve-out (precedence, explicit):
+torchrun / torch.distributed.elastic wraps *every* worker death — transient
+crashes included — in a `ChildFailedError` and prints ITS OWN traceback; by
+default the child's real traceback is not in this log (`error_file: <N/A>`).
+That wrapper terminal line is `…errors.ChildFailedError:` — `ChildFailedError`
+is not in the deterministic marker set, so rule 2 never fires on the wrapper.
+The wrapper's bare "Traceback" line WOULD trip rule 3, so rule 3 is
+disqualified whenever `ChildFailedError` is present, letting the death fall
+through to the transient default (rule 4) — which is exactly what the
+kill-and-resume e2e depends on. Rule 2 is intentionally NOT gated by the
+wrapper: a genuine child terminal line (e.g. a real `ModuleNotFoundError:` if
+elastic error-files are enabled) is a true deterministic bug and should fail
+fast even under torchrun.
+
+Deliberately narrow: it never fabricates node / accelerator / communication /
+storage signals a single local process cannot actually evidence — those
+classes belong to the distributed coordinator, not the local launcher.
+"""
+
+from __future__ import annotations
+
+import re
+
+from flashruntime.recovery.taxonomy import FailureSignals
+
+# Exception-type names that mark a DETERMINISTIC bug: the same failure recurs
+# byte-for-byte on retry, so retrying only re-reaches it.
+_DETERMINISTIC_EXC_MARKERS = (
+    "SyntaxError",          # the file will not parse — a retry parses the same file
+    "IndentationError",     # a parse error too — same input, same failure
+    "ImportError",          # a missing/broken dependency does not heal on retry
+    "ModuleNotFoundError",  # ImportError's subclass; its name lacks the substring "ImportError"
+    "NameError",            # an undefined name is a code bug, not bad luck
+    "AttributeError",       # a code / version-contract bug, deterministic by nature
+)
+
+# A marker only counts when it stands where CPython prints the failing type: at
+# the START of the traceback's final line, either bare (`NameError`) or with a
+# message (`NameError: ...`), optionally dotted with the defining module
+# (`pkg.mod.NameError: ...`). Line-anchoring (see module docstring) is the fix
+# for the incidental-substring false positive — exception names in prose don't
+# start lines; traceback terminals always do. re.MULTILINE makes ^/$ match at
+# every line boundary; the trailing `(?::|$)` boundary stops a marker from
+# matching a longer name it merely prefixes (e.g. `NameErrorX:`).
+_DETERMINISTIC_TERMINAL_RE = re.compile(
+    r"^(?:\w+\.)*(?:" + "|".join(_DETERMINISTIC_EXC_MARKERS) + r")(?::|$)",
+    re.MULTILINE,
+)
+
+# torchrun / elastic re-raises ChildFailedError (with its own traceback) for
+# ANY worker death, transient ones included — see the module docstring. Its
+# presence disqualifies the bare-traceback rule below.
+_ELASTIC_WRAPPER_MARKER = "ChildFailedError"
+_TRACEBACK_MARKER = "Traceback (most recent call last):"
+
+
+def from_local_launch(exit_code: int | None, log_tail: str) -> FailureSignals:
+    """Map a finished local process to the signals `classify()` reads.
+
+    `exit_code` is the child's OS return code (None if it never started);
+    `log_tail` is the tail of its captured stdout+stderr. Rules are checked
+    top-to-bottom, first match wins — see the module docstring for the full
+    table and the reasoning behind each class.
+    """
+    log = log_tail or ""
+
+    # Rule 1 — a clean exit is not a failure; nothing to recover. Guard only:
+    # the retry loop never asks on SUCCEEDED, but this keeps the function total.
+    if exit_code == 0 or exit_code is None:
+        return FailureSignals(exit_code=exit_code)
+
+    # Rule 2 — a named deterministic bug (import / parse / name error) standing
+    # on a traceback TERMINAL line: the same error reappears on a byte-identical
+    # retry, so mark it deterministic and let the policy fail fast. Anchored to
+    # the line start (not a bare substring) so an exception name mentioned in a
+    # prose log line — e.g. `… WARNING ImportError: flash_attn not available` —
+    # is not mistaken for the failure that actually killed the process.
+    if _DETERMINISTIC_TERMINAL_RE.search(log):
+        return FailureSignals(exit_code=exit_code, exit_deterministic=True)
+
+    # Rule 3 — an unhandled exception that printed its own traceback:
+    # deterministic by default (unhandled exceptions are code errors). Excluded
+    # when it is the torchrun elastic wrapper, whose traceback is the launcher's
+    # stack and says nothing about whether the user's failure was deterministic.
+    if _TRACEBACK_MARKER in log and _ELASTIC_WRAPPER_MARKER not in log:
+        return FailureSignals(exit_code=exit_code, exit_deterministic=True)
+
+    # Rule 4 — everything else: a signal death / OOM (SIGKILL/SIGSEGV, exit 137,
+    # a negative POSIX returncode), a bare SystemExit with no traceback, or a
+    # torchrun ChildFailedError — all transient. Leave exit_deterministic False
+    # so classify() returns WORKER_CRASH and the policy grants a fresh attempt.
+    return FailureSignals(exit_code=exit_code)