flashruntime 0.3.0__py3-none-any.whl

flashruntime/checkpoint/local.py

@@ -0,0 +1,109 @@
+"""Local checkpoint manifests: parts-first / manifest-last for processes
+that have only a filesystem (no coordinator).
+
+`write_manifest` hashes every part file already on disk and writes
+manifest.json LAST — a crash mid-checkpoint leaves part files but no
+manifest, so the checkpoint does not exist. `latest_valid_manifest`
+re-verifies every part hash on read: a corrupted or truncated part
+disqualifies its manifest, so recovery can never restore from it.
+
+Consumers: `flashruntime.torch.checkpoint()` and the Hugging Face Trainer
+callback. Pure stdlib + pydantic (protocol models) — safe in the core.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from pathlib import Path
+
+from flashruntime.protocol.v1alpha1 import (
+    CheckpointManifest,
+    CheckpointPart,
+    CheckpointValidation,
+)
+
+MANIFEST_NAME = "manifest.json"
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_manifest(
+    step_dir: Path,
+    *,
+    job_id: str,
+    attempt_id: str,
+    step: int,
+    world_size: int = 1,
+    framework: str = "",
+) -> CheckpointManifest:
+    """Hash every part file in `step_dir`, then write the manifest LAST."""
+    step_dir = Path(step_dir)
+    parts = [
+        CheckpointPart(key=p.name, sha256=_sha256(p), size_bytes=p.stat().st_size)
+        for p in sorted(step_dir.iterdir())
+        if p.is_file() and p.name != MANIFEST_NAME
+    ]
+    if not parts:
+        raise ValueError(f"no part files in {step_dir}")
+    manifest = CheckpointManifest(
+        manifest_id=f"ck-{uuid.uuid4().hex[:12]}",
+        job_id=job_id,
+        attempt_id=attempt_id,
+        step=step,
+        framework=framework,
+        world_size=world_size,
+        compatible_world_sizes=[world_size],
+        storage_prefix=str(step_dir),
+        parts=parts,
+        validation=CheckpointValidation.HASH_VERIFIED,
+    )
+    (step_dir / MANIFEST_NAME).write_text(manifest.model_dump_json(indent=2))  # LAST
+    return manifest
+
+
+def verify_manifest(manifest: CheckpointManifest, step_dir: Path) -> bool:
+    """True iff every part named in `manifest` is present in `step_dir` and
+    re-hashes to its recorded sha256 — i.e. this checkpoint is safe to
+    restore. The single source of truth for that question: both
+    `latest_valid_manifest` (recovery's picker) and the viewer's per-step
+    listing call it, so the part-hashing lives in exactly one place and can
+    never drift between the two. Never raises: a part we cannot re-hash
+    (directory, unreadable, gone) means the manifest cannot be verified, so
+    it is simply invalid."""
+    step_dir = Path(step_dir)
+    try:
+        return all(
+            (step_dir / part.key).is_file() and _sha256(step_dir / part.key) == part.sha256
+            for part in manifest.parts
+        )
+    except (OSError, ValueError):
+        return False
+
+
+def latest_valid_manifest(ckpt_root: Path, pattern: str = "step-*") -> CheckpointManifest | None:
+    """Newest manifest whose parts all re-verify on disk, or None.
+
+    `pattern` matches the per-step directory names ("step-*" for
+    flashruntime.torch, "checkpoint-*" for Hugging Face Trainer output).
+    """
+    ckpt_root = Path(ckpt_root)
+    if not ckpt_root.is_dir():
+        return None
+    best: CheckpointManifest | None = None
+    for mf_path in ckpt_root.glob(f"{pattern}/{MANIFEST_NAME}"):
+        try:
+            manifest = CheckpointManifest.model_validate_json(mf_path.read_text())
+        except (OSError, ValueError):
+            # unreadable/invalid manifest (bad JSON, or manifest.json is a
+            # directory / unreadable file → OSError): treat as nonexistent
+            continue
+        if verify_manifest(manifest, mf_path.parent) and (best is None or manifest.step > best.step):
+            best = manifest
+    return best

flashruntime/checkpoint/store.py

@@ -0,0 +1,86 @@
+"""Manifest persistence seam (research item R1).
+
+Today `CheckpointCatalog` keeps manifests in a plain dict — a coordinator
+restart orphans perfectly good checkpoint *files* because the metadata
+proving their validity dies with the process. This module defines the
+persistence interface that closes that gap, mirroring the lease pattern
+that already works (`leases/store.py` + `leases/sqlite_store.py`):
+
+    the catalog mutates live objects in an insertion-ordered cache and
+    calls `save(manifest)` after every transition; a durable store
+    persists there and rehydrates the cache at construction.
+
+Migration path (half a day, SPRINT_PLAN Day 3):
+1. `CheckpointCatalog.__init__` grows `store: ManifestStore | None = None`
+   (default `InMemoryManifestStore()` — zero behavior change).
+2. The catalog's `_manifests` dict is replaced by the store's cache, and
+   every mutation (`commit`, `mark_restore_verified`, `quarantine`) ends
+   with `store.save(manifest)`.
+3. `SqliteManifestStore` lands beside `leases.db` (one table: manifest_id
+   PK, scope job_id, manifest JSON, seq for order) with the same
+   COALESCE-seq upsert as `SqliteLeaseStore._persist` — copy that code,
+   it is tested and correct.
+4. The conformance test is `tests/test_checkpoint.py` re-run against the
+   new store, plus a restart-survival test shaped exactly like
+   `tests/test_leases_sqlite.py::test_inflight_work_survives_coordinator_restart`.
+
+Note the pending-parts registry (`CheckpointCatalog._registered`) is
+deliberately NOT persisted: parts without a committed manifest are, by the
+manifest-last rule, not checkpoints — losing them on restart just means
+the in-flight checkpoint re-uploads, which the relay already handles.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from flashruntime.protocol.v1alpha1 import CheckpointManifest
+
+__all__ = ["ManifestStore", "InMemoryManifestStore"]
+
+
+class ManifestStore(Protocol):
+    """Minimal persistence contract for checkpoint manifests.
+
+    Semantics (identical in spirit to `LeaseStore`):
+    - `add` registers a NEW manifest (duplicate manifest_id → ValueError);
+    - `save` persists the current state of an already-added manifest
+      (validation upgrades/quarantine); no-op for in-memory stores whose
+      cache holds live references;
+    - `get`/`all` read from the cache — `all(scope)` filters by the
+      catalog's scope key (the service uses `f"{job_id}::{task_id}"`) and
+      MUST preserve insertion order (selection tie-breaks depend on it).
+    """
+
+    def add(self, manifest: CheckpointManifest) -> None: ...
+
+    def save(self, manifest: CheckpointManifest) -> None: ...
+
+    def get(self, manifest_id: str) -> CheckpointManifest | None: ...
+
+    def all(self, scope: str | None = None) -> list[CheckpointManifest]: ...
+
+
+class InMemoryManifestStore:
+    """Reference implementation: live references, insertion-ordered,
+    volatile. This is functionally what the catalog does today — extracted
+    behind the seam so the SQLite store is a drop-in."""
+
+    def __init__(self) -> None:
+        self._manifests: dict[str, CheckpointManifest] = {}
+
+    def add(self, manifest: CheckpointManifest) -> None:
+        if manifest.manifest_id in self._manifests:
+            raise ValueError(f"manifest {manifest.manifest_id} already exists")
+        self._manifests[manifest.manifest_id] = manifest
+
+    def save(self, manifest: CheckpointManifest) -> None:
+        pass  # live references — mutations already visible
+
+    def get(self, manifest_id: str) -> CheckpointManifest | None:
+        return self._manifests.get(manifest_id)
+
+    def all(self, scope: str | None = None) -> list[CheckpointManifest]:
+        return [
+            m for m in self._manifests.values() if scope is None or m.job_id == scope
+        ]

flashruntime/integrations/__init__.py

@@ -0,0 +1,5 @@
+"""Framework adapters: each builds CommandWorkloads from one framework's
+LAUNCH AND CHECKPOINT CONVENTIONS — never its model code, never a
+module-level framework import (four-axes rule). Import the submodule you
+need: `from flashruntime.integrations import sklearn, pytorch, huggingface`.
+"""

flashruntime/integrations/huggingface.py

@@ -0,0 +1,59 @@
+"""Hugging Face adapter: HF Trainer already wraps DDP/FSDP internally when
+launched by torchrun, so launching is the pytorch adapter's job. What HF
+adds is its callback seam — `flashruntime_callback()` commits Trainer
+checkpoints as verified manifests. transformers is imported only inside
+that factory, in the user's training process.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from flashruntime.checkpoint.local import latest_valid_manifest
+from flashruntime.integrations.pytorch import ddp
+from flashruntime.workloads.command import CommandWorkload
+
+
+def trainer(
+    script: str, *, source: str = ".", nproc_per_node: int = 1, script_args: str = ""
+) -> CommandWorkload:
+    return ddp(script, source=source, nproc_per_node=nproc_per_node, script_args=script_args)
+
+
+def latest_checkpoint(output_dir: str | Path) -> str | None:
+    """Newest Trainer checkpoint dir with a VALID manifest — pass as
+    `trainer.train(resume_from_checkpoint=...)`. None means fresh start."""
+    manifest = latest_valid_manifest(Path(output_dir), pattern="checkpoint-*")
+    return None if manifest is None else manifest.storage_prefix
+
+
+def flashruntime_callback():
+    """Build the TrainerCallback (transformers import paid here, in the
+    user's process only): on_save commits a manifest, on_log relays metrics."""
+    from transformers import TrainerCallback  # noqa: PLC0415 — user process only
+
+    from flashruntime.checkpoint.local import write_manifest
+
+    class FlashRuntimeCallback(TrainerCallback):
+        def on_save(self, args, state, control, **kwargs):
+            if state.is_world_process_zero:
+                step_dir = Path(args.output_dir) / f"checkpoint-{state.global_step}"
+                if step_dir.is_dir():
+                    write_manifest(
+                        step_dir,
+                        job_id=os.environ.get("FLASHML_JOB_ID", "local"),
+                        attempt_id=os.environ.get("FLASHML_ATTEMPT_ID", "local"),
+                        step=state.global_step,
+                        framework="transformers",
+                    )
+            return control
+
+        def on_log(self, args, state, control, logs=None, **kwargs):
+            if logs and state.is_world_process_zero:
+                from flashruntime.torch import log_metrics
+
+                log_metrics({**logs, "step": state.global_step})
+            return control
+
+    return FlashRuntimeCallback()

flashruntime/integrations/pytorch.py

@@ -0,0 +1,52 @@
+"""PyTorch adapter: launch conventions only. torchrun starts N processes
+and hands each RANK/WORLD_SIZE — the user's code (or
+flashruntime.torch.prepare) wires DDP from there. No torch import here.
+"""
+
+from __future__ import annotations
+
+import shlex
+
+from flashruntime.protocol.plan_v1alpha1 import CheckpointPolicy
+from flashruntime.workloads.command import CommandWorkload, OutputSpec, Source
+
+
+def ddp(
+    script: str,
+    *,
+    source: str = ".",
+    nproc_per_node: int = 2,
+    nnodes: int = 1,
+    script_args: str = "",
+    env: dict[str, str] | None = None,
+) -> CommandWorkload:
+    if nnodes > 1:
+        raise NotImplementedError(
+            "multi-node rendezvous is a launcher concern — later slice (spec §10); "
+            "--standalone below is single-node by definition"
+        )
+    command = [
+        "torchrun",
+        f"--nproc-per-node={nproc_per_node}",
+        f"--nnodes={nnodes}",
+        "--standalone",
+        # Single-node by definition (nnodes > 1 raised above), so pin the
+        # advertised rendezvous address. Without this torchrun advertises
+        # socket.getfqdn(), which on some macOS DNS setups returns an
+        # unresolvable ip6.arpa name — workers then retry DNS forever and
+        # the run hangs before spawning a single process.
+        "--local-addr=127.0.0.1",
+        script,
+        *shlex.split(script_args),
+    ]
+    return CommandWorkload(
+        command=command,
+        source=Source(path=source),
+        env=env or {},
+        mode="coordinated",
+        checkpoint=CheckpointPolicy(
+            backend="local_manifest",
+            note="flashruntime.torch.checkpoint: parts-first/manifest-last under FLASHML_CKPT_DIR",
+        ),
+        outputs=OutputSpec(collect=["metrics.json"]),
+    )

flashruntime/integrations/sklearn.py

@@ -0,0 +1,42 @@
+"""sklearn adapter: distribute across runs, never inside .fit().
+
+The contract with the user's script is pure convention: CLI flags in,
+metrics.json out. No sklearn import here — the script owns the estimator.
+"""
+
+from __future__ import annotations
+
+import itertools
+
+from flashruntime.workloads.command import CommandWorkload, OutputSpec, Source
+
+
+def sweep(
+    script: str,
+    task_params: list[dict],
+    *,
+    source: str = ".",
+    metric: str = "accuracy_mean",
+    maximize: bool = True,
+    python: str = "python",
+) -> CommandWorkload:
+    """One independent task per params dict. Every dict must carry every
+    key (the CLI flags are built from the union)."""
+    keys = sorted({k for p in task_params for k in p})
+    command = [python, script]
+    for key in keys:
+        command += [f"--{key}", "{" + key + "}"]
+    return CommandWorkload(
+        command=command,
+        source=Source(path=source),
+        task_params=task_params,
+        mode="independent_tasks",
+        outputs=OutputSpec(collect=["metrics.json"], primary_metric=metric, maximize=maximize),
+    )
+
+
+def hpo(script: str, grid: dict[str, list], **kwargs) -> CommandWorkload:
+    """Cartesian grid search: {"model": ["logreg","rf"], "C": [0.1, 1]} → 4 trials."""
+    keys = sorted(grid)
+    trials = [dict(zip(keys, combo)) for combo in itertools.product(*(grid[k] for k in keys))]
+    return sweep(script, trials, **kwargs)

flashruntime/launchers/__init__.py

@@ -0,0 +1,130 @@
+"""Launchers: start a compiled `LaunchSpec` and watch it run.
+
+The launcher owns everything environment-shaped that compilation must not
+touch: resolving rendezvous addresses/ports, materializing `spec.files`
+into a workdir, injecting credentials, spawning the processes (locally,
+via torchrun, on Kubernetes, on Slurm), and reporting their fate.
+
+Division of labor (the four-axes rule, HANDBOOK §2.1):
+  compiler = *what* to run · launcher = *how processes start* ·
+  provider = *where machines come from* · recipe = *what the user's code is*.
+
+Failure semantics: a launcher NEVER retries or recovers — it reports.
+Recovery is the coordinator's job (`recovery.decide()` chooses
+RESTART_GROUP etc.); a launcher that silently relaunched would corrupt the
+ledger's story and double-spend the user's budget.
+
+Status: interface complete (final surface); `TorchrunLauncher` is the
+first concrete (SPRINT_PLAN Week 2), then kubernetes/slurm/skypilot per
+demand — see FLASHRUNTIME_EVALUATION §J for each substrate's role.
+"""
+
+from __future__ import annotations
+
+import enum
+from abc import ABC, abstractmethod
+from typing import ClassVar
+
+from flashruntime.strategies import LaunchSpec
+
+__all__ = ["LaunchState", "LaunchHandle", "Launcher", "LaunchError"]
+
+
+class LaunchError(Exception):
+    """Launch could not start (preflight or spawn failure). Once processes
+    are running, failures are reported through `LaunchHandle.poll`, never
+    raised — the coordinator must always get a story, not a stack trace."""
+
+
+class LaunchState(str, enum.Enum):
+    """Coarse lifecycle every substrate can map onto (K8s pod phases,
+    process exit codes, Slurm job states all reduce to these five)."""
+
+    PENDING = "PENDING"      # accepted, processes not yet running
+    RUNNING = "RUNNING"      # at least one worker alive, none failed
+    SUCCEEDED = "SUCCEEDED"  # all workers exited 0
+    FAILED = "FAILED"        # any worker exited non-zero / infra killed it
+    CANCELLED = "CANCELLED"  # cancel() honored
+
+    @property
+    def terminal(self) -> bool:
+        return self in (LaunchState.SUCCEEDED, LaunchState.FAILED, LaunchState.CANCELLED)
+
+
+class LaunchHandle(ABC):
+    """A live (or finished) launch. Handles are cheap views over substrate
+    state — polling must be side-effect free and safe to call forever."""
+
+    @abstractmethod
+    def poll(self) -> LaunchState:
+        """Current state. Must be non-blocking and idempotent. After a
+        terminal state is returned once, every later call returns the same
+        terminal state (substrates that garbage-collect finished jobs must
+        cache the outcome)."""
+
+    @abstractmethod
+    def cancel(self) -> None:
+        """Best-effort stop: idempotent, non-blocking, never raises on an
+        already-terminal launch. The next `poll()` after a successful
+        cancel eventually reports CANCELLED (grace periods allowed)."""
+
+    def wait(self, timeout_seconds: float | None = None) -> LaunchState:
+        """Convenience: poll until terminal or timeout; returns the last
+        observed state (callers check `.terminal`). Default implementation
+        polls at 1 s; substrates with native wait primitives override."""
+        import time
+
+        deadline = None if timeout_seconds is None else time.monotonic() + timeout_seconds
+        while True:
+            state = self.poll()
+            if state.terminal or (deadline and time.monotonic() >= deadline):
+                return state
+            time.sleep(1.0)
+
+    def logs(self, tail_lines: int = 200) -> str:
+        """Recent combined output for the ledger/dashboard. Optional —
+        default is an honest empty string, never fabricated content."""
+        return ""
+
+    @property
+    def execution_id(self) -> str:
+        """Substrate-native identifier (pid list, RayJob name, Slurm job
+        id) for the ledger's `runtime_execution_id`. Default: unnamed."""
+        return ""
+
+
+class Launcher(ABC):
+    """One way of starting process groups.
+
+    Contract:
+    - `healthy()` is the ONLY place environment preflight lives (binary on
+      PATH, cluster reachable, quota available). Compilers stay pure.
+    - `launch()` must either raise `LaunchError` before anything started,
+      or return a handle — never leave orphaned half-started state without
+      a handle that can cancel it.
+    - Secrets/credentials are injected here (from the launcher's own
+      config), merged UNDER `spec.env` so a plan can never override
+      security-relevant variables.
+    """
+
+    #: Name matching `StrategyPlan.launcher` values
+    #: ("local", "torchrun", "kubernetes", "slurm", "skypilot",
+    #:  "flashruntime-leases").
+    name: ClassVar[str]
+
+    def healthy(self) -> tuple[bool, str]:
+        """Preflight: can this launcher launch right now? Returns
+        (ok, human-readable reason). Called before spending money; a False
+        here routes the coordinator to PAUSE, not to a doomed launch."""
+        return True, "no preflight implemented"
+
+    @abstractmethod
+    def launch(self, spec: LaunchSpec, job_id: str, attempt_id: str) -> LaunchHandle:
+        """Start the group described by `spec`.
+
+        Inputs: the compiled spec; job/attempt ids for labeling substrate
+        resources (the flashml.dev/* label convention) so operators can
+        trace any pod/process back to its ledger entry.
+        Output: a `LaunchHandle` (see its contract).
+        Raises: `LaunchError` only for failures *before* anything runs.
+        """

flashruntime/launchers/local.py

@@ -0,0 +1,126 @@
+"""LocalProcessLauncher — the first concrete Launcher.
+
+Runs a LaunchSpec as one OS process on this machine: cwd from
+`workdir_hint`, the caller's environment merged UNDER `spec.env` and the
+FlashRuntime contract variables, stdout+stderr captured to a log file in
+the attempt's output directory. This is Mode 0 execution and the substrate
+under `flash.submit(...)`'s local path.
+
+Contract variables exported to the child (opt-in for user code):
+  FLASHML_OUTPUT_DIR  — per-attempt scratch/output directory
+  FLASHML_CKPT_DIR    — per-JOB checkpoint tree (attempts share it, so a
+                        restarted attempt can restore its predecessor's
+                        manifests — the resume path depends on this)
+  FLASHML_JOB_ID / FLASHML_ATTEMPT_ID
+
+Honors the Launcher contract: LaunchError only before a process exists;
+after that, every failure is reported through poll(), never raised — and
+this launcher never retries (recovery belongs to the coordinator).
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+from flashruntime.launchers import Launcher, LaunchError, LaunchHandle, LaunchState
+from flashruntime.strategies import LaunchSpec
+
+
+class LocalLaunchHandle(LaunchHandle):
+    def __init__(self, proc: subprocess.Popen, log_path: Path, output_dir: Path):
+        self._proc = proc
+        self._log_path = log_path
+        self.output_dir = output_dir
+        self._final: LaunchState | None = None
+        self._cancelled = False
+
+    def poll(self) -> LaunchState:
+        if self._final is not None:
+            return self._final
+        code = self._proc.poll()
+        if code is None:
+            return LaunchState.RUNNING
+        if self._cancelled:
+            self._final = LaunchState.CANCELLED
+        else:
+            self._final = LaunchState.SUCCEEDED if code == 0 else LaunchState.FAILED
+        return self._final
+
+    def cancel(self) -> None:
+        if self.poll().terminal:
+            return
+        self._cancelled = True
+        self._proc.terminate()
+
+    def wait(self, timeout_seconds: float | None = None) -> LaunchState:
+        # native wait beats the ABC's 1 s polling loop
+        try:
+            self._proc.wait(timeout=timeout_seconds)
+        except subprocess.TimeoutExpired:
+            pass
+        return self.poll()
+
+    def logs(self, tail_lines: int = 200) -> str:
+        if not self._log_path.is_file():
+            return ""
+        lines = self._log_path.read_text(errors="replace").splitlines()
+        return "\n".join(lines[-tail_lines:])
+
+    @property
+    def execution_id(self) -> str:
+        return str(self._proc.pid)
+
+    @property
+    def exit_code(self) -> int | None:
+        """The child's raw OS return code once terminal (None while running).
+        The coarse LaunchState collapses every nonzero exit to FAILED, but
+        recovery needs the number: 137/-9 (signal death) and a bare SystemExit
+        classify differently from an ImportError traceback. Populated by the
+        preceding poll()/wait(); a negative value is a POSIX signal number."""
+        return self._proc.returncode
+
+
+class LocalProcessLauncher(Launcher):
+    name = "local"
+
+    def __init__(self, output_root: str | Path):
+        self._output_root = Path(output_root)
+
+    def launch(self, spec: LaunchSpec, job_id: str, attempt_id: str) -> LocalLaunchHandle:
+        workdir = Path(spec.workdir_hint or ".").expanduser()
+        if not spec.argv:
+            raise LaunchError("empty argv")
+        if not workdir.is_dir():
+            raise LaunchError(f"workdir does not exist: {workdir}")
+        outdir = self._output_root / job_id / attempt_id
+        outdir.mkdir(parents=True, exist_ok=True)
+        for name, content in spec.files.items():
+            (outdir / name).write_text(content)
+        env = {
+            **os.environ,
+            **spec.env,
+            "FLASHML_OUTPUT_DIR": str(outdir),
+            "FLASHML_CKPT_DIR": str(self._output_root / job_id / "ckpt"),
+            "FLASHML_JOB_ID": job_id,
+            "FLASHML_ATTEMPT_ID": attempt_id,
+        }
+        log_path = outdir / "launcher.log"
+        # Popen dups the fd into the child, so the parent's handle can (and
+        # must) be closed immediately — leaving it open leaks a descriptor
+        # and trips ResourceWarning; logs() re-reads the file from disk.
+        log_file = open(log_path, "w")
+        try:
+            proc = subprocess.Popen(
+                spec.argv,
+                cwd=str(workdir),
+                env=env,
+                stdout=log_file,
+                stderr=subprocess.STDOUT,
+            )
+        except OSError as exc:
+            raise LaunchError(f"failed to start {spec.argv[0]!r}: {exc}") from exc
+        finally:
+            log_file.close()
+        return LocalLaunchHandle(proc, log_path, outdir)

flashruntime/leases/__init__.py

@@ -0,0 +1,27 @@
+"""Task lease semantics: assignment, renewal via heartbeat, expiration.
+
+A lease grants one node the right to execute a task attempt for a bounded
+period. Missed heartbeats past the deadline expire the lease and the task
+is reassigned. Only one attempt may ever commit — late duplicates are
+rejected (and recorded).
+
+Embeddable, pure-Python, no I/O:
+
+    from flashruntime.leases import LeaseManager
+    from flashruntime.protocol.v1alpha1 import TaskSpec
+
+    mgr = LeaseManager(on_event=print)
+    mgr.add_task(TaskSpec(task_id="t1", job_id="j1", commit_key="j1/t1"))
+    lease = mgr.claim(node_id="laptop-1")
+    ... work ...
+    mgr.heartbeat(lease.lease_id)
+    mgr.complete(lease.lease_id, output_sha256="...")
+
+The FlashRuntime service exposes this same manager over HTTP; flashnode's
+device executor is its remote client.
+"""
+
+from flashruntime.leases.manager import LeaseError, LeaseManager
+from flashruntime.leases.store import InMemoryLeaseStore, LeaseStore, TaskRecord
+
+__all__ = ["LeaseManager", "LeaseError", "LeaseStore", "InMemoryLeaseStore", "TaskRecord"]