autoforge-trainer 1.0.2__py3-none-any.whl

autoforge/cli/runs.py

@@ -0,0 +1,233 @@
+"""`autoforge run ...` and `autoforge plan ...` commands."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import typer
+
+from autoforge.cli.output import JsonOpt, emit, errors_as_envelope, fail
+from autoforge.engine import runner
+from autoforge.engine.store import TERMINAL, RunStore
+from autoforge.engine.validate import check_plan, load_plan_file
+
+run_app = typer.Typer(help="Start and control training runs.", no_args_is_help=True)
+plan_app = typer.Typer(help="Check training plans.", no_args_is_help=True)
+
+ActorOpt = typer.Option(
+    None, "--actor", help="Who is acting: 'human' or 'ai:<name>'. Defaults to $AUTOFORGE_ACTOR or 'human'."
+)
+
+
+def _actor(actor: str | None) -> str:
+    return actor or os.environ.get("AUTOFORGE_ACTOR", "human")
+
+
+def _view_line(v: runner.RunView) -> str:
+    m = v.metrics or v.latest_metrics
+    shown = ", ".join(f"{k}={val:.4g}" for k, val in m.items())
+    return f"{v.run_id:<40} {v.status.value:<10} {v.recipe or 'custom':<20} {shown}"
+
+
+def _follow(store: RunStore, timeout: float | None) -> runner.RunView:
+    """Print metric/log events until the run finishes (human mode for --wait)."""
+    seen = 0
+    end = time.monotonic() + timeout if timeout else None
+    while True:
+        events = store.events()
+        for e in events[seen:]:
+            if e.get("type") == "metric":
+                step = f"step {e['step']}: " if e.get("step") is not None else ""
+                typer.echo(f"  {step}{e['name']} = {e['value']:.4g}")
+            elif e.get("type") == "log":
+                typer.echo(f"  {e['message']}")
+        seen = len(events)
+        view = runner.status(store)
+        if view.status in TERMINAL or (end and time.monotonic() > end):
+            return view
+        time.sleep(0.3)
+
+
+@plan_app.command("validate")
+def plan_validate(plan_file: Path, as_json: bool = JsonOpt) -> None:
+    """Check a plan against installed packs, data and hardware. Nothing is run."""
+    with errors_as_envelope(as_json):
+        plan = load_plan_file(plan_file)
+    check = check_plan(plan)
+    if not check.ok:
+        fail(as_json, "PLAN_INVALID", f"{len(check.problems)} problem(s) in plan",
+             "Fix every item in details.problems.", check.model_dump())
+    human = "\n".join([f"OK: device={check.device}, entrypoint={check.entrypoint}",
+                       *(f"warning: {w}" for w in check.warnings)])
+    emit(as_json, check.model_dump(), human)
+
+
+@run_app.command("start")
+def start(
+    plan_file: Path,
+    wait: bool = typer.Option(False, "--wait", help="Block until the run finishes."),
+    timeout: float = typer.Option(None, "--timeout", help="Max seconds to wait with --wait."),
+    actor: str = ActorOpt,
+    as_json: bool = JsonOpt,
+) -> None:
+    """Validate a plan and start training in the background. Returns the run_id immediately."""
+    with errors_as_envelope(as_json):
+        plan = load_plan_file(plan_file)
+        store = runner.create_run(plan, _actor(actor))
+        runner.start(store, _actor(actor))
+    if not wait:
+        view = runner.status(store)
+        emit(as_json, view.model_dump(mode="json"), f"Started {store.run_id}\nPoll with: autoforge run status {store.run_id}")
+        return
+    if as_json:
+        view = runner.wait(store, timeout)
+    else:
+        typer.echo(f"Started {store.run_id}")
+        view = _follow(store, timeout)
+    emit(as_json, view.model_dump(mode="json"), _view_line(view) + (f"\nerror: {view.error}" if view.error else ""))
+
+
+@run_app.command("status")
+def status(run_id: str, as_json: bool = JsonOpt) -> None:
+    """Current status, final metrics and latest logged metrics of a run."""
+    with errors_as_envelope(as_json):
+        view = runner.status(RunStore.open(run_id))
+    emit(as_json, view.model_dump(mode="json"), _view_line(view) + (f"\nerror: {view.error}" if view.error else ""))
+
+
+@run_app.command("list")
+def list_cmd(as_json: bool = JsonOpt) -> None:
+    """All runs, newest first."""
+    views = runner.all_runs()
+    emit(as_json, [v.model_dump(mode="json") for v in views], "\n".join(map(_view_line, views)) or "(no runs)")
+
+
+def _control(action: str, run_id: str, actor: str | None, as_json: bool, **kw) -> None:
+    with errors_as_envelope(as_json):
+        store = RunStore.open(run_id)
+        view = getattr(runner, action)(store, _actor(actor), **kw)
+    emit(as_json, view.model_dump(mode="json"), _view_line(view))
+
+
+@run_app.command("pause")
+def pause(run_id: str, actor: str = ActorOpt, as_json: bool = JsonOpt) -> None:
+    """Suspend a running run (works with any trainer)."""
+    _control("pause", run_id, actor, as_json)
+
+
+@run_app.command("resume")
+def resume(run_id: str, actor: str = ActorOpt, as_json: bool = JsonOpt) -> None:
+    """Continue a paused run."""
+    _control("resume", run_id, actor, as_json)
+
+
+@run_app.command("stop")
+def stop(
+    run_id: str,
+    grace: float = typer.Option(10.0, "--grace", help="Seconds to wait for a clean stop before terminating."),
+    actor: str = ActorOpt,
+    as_json: bool = JsonOpt,
+) -> None:
+    """Ask the run to stop; terminate it if it does not stop within the grace period."""
+    _control("stop", run_id, actor, as_json, grace_seconds=grace)
+
+
+@run_app.command("retry")
+def retry(
+    run_id: str,
+    wait: bool = typer.Option(False, "--wait"),
+    actor: str = ActorOpt,
+    as_json: bool = JsonOpt,
+) -> None:
+    """Start a new run with the same plan, resuming from the old run's checkpoints if any."""
+    with errors_as_envelope(as_json):
+        store = runner.retry(RunStore.open(run_id), _actor(actor))
+        runner.start(store, _actor(actor))
+    view = runner.wait(store) if wait else runner.status(store)
+    emit(as_json, view.model_dump(mode="json"), f"Started {store.run_id} (retry of {run_id})")
+
+
+@run_app.command("logs")
+def logs(run_id: str, tail: int = typer.Option(50, "--tail"), as_json: bool = JsonOpt) -> None:
+    """Last lines of the worker's stdout/stderr."""
+    with errors_as_envelope(as_json):
+        lines = RunStore.open(run_id).logs_tail(tail)
+    emit(as_json, {"lines": lines}, "\n".join(lines))
+
+
+@run_app.command("metrics")
+def metrics(run_id: str, last: int = typer.Option(5, "--last"), as_json: bool = JsonOpt) -> None:
+    """Per-metric summary: last, min, max, count and the most recent values."""
+    with errors_as_envelope(as_json):
+        summary = runner.metrics_summary(RunStore.open(run_id), last)
+    human = "\n".join(
+        f"{k:<20} last={v['last']:.4g} min={v['min']:.4g} max={v['max']:.4g} n={v['count']}"
+        for k, v in summary.items()
+    ) or "(no metrics yet)"
+    emit(as_json, summary, human)
+
+
+@run_app.command("audit")
+def audit(run_id: str, as_json: bool = JsonOpt) -> None:
+    """Every action taken on a run and who took it."""
+    with errors_as_envelope(as_json):
+        rows = RunStore.open(run_id).audit_log()
+    emit(as_json, rows, "\n".join(f"{r['t']}  {r['actor']:<16} {r['action']}" for r in rows))
+
+
+@run_app.command("diagnose")
+def diagnose(run_id: str, as_json: bool = JsonOpt) -> None:
+    """Findings and suggestions: failure cause, diverging/plateauing loss, overfitting, leakage."""
+    from autoforge import api
+
+    with errors_as_envelope(as_json):
+        d = api.run_diagnose(run_id)
+    lines = [d["summary"]] + [f"  [{f['severity']}] {f['code']}: {f['message']}" +
+                              (f"\n      → {f['suggestion']}" if f["suggestion"] else "") for f in d["findings"]]
+    emit(as_json, d, "\n".join(lines))
+
+
+@run_app.command("adjust")
+def adjust(
+    run_id: str,
+    set_: list[str] = typer.Option(..., "--set", help="name=value (YAML value), repeatable. e.g. --set lr=0.001"),
+    actor: str = ActorOpt,
+    as_json: bool = JsonOpt,
+) -> None:
+    """Change hyperparameters of a running run (trainer must call ctx.check_updates())."""
+    import yaml
+
+    from autoforge import api
+
+    changes = {}
+    for item in set_:
+        name, sep, value = item.partition("=")
+        if not sep or not name:
+            fail(as_json, "BAD_REQUEST", f"bad --set '{item}'", "use name=value")
+        changes[name.strip()] = yaml.safe_load(value)
+    with errors_as_envelope(as_json):
+        out = api.run_adjust(run_id, _actor(actor), changes)
+    emit(as_json, out, f"Sent {changes} to {run_id} (seq {out['seq']})")
+
+
+@run_app.command("compare")
+def compare(
+    run_ids: list[str],
+    metric: str = typer.Option(None, "--metric", "-m", help="Sort by this metric, best first."),
+    as_json: bool = JsonOpt,
+) -> None:
+    """Compare runs: metrics and the hyperparameters that differ."""
+    from autoforge import api
+
+    with errors_as_envelope(as_json):
+        c = api.run_compare(run_ids, metric)
+    diff = c["differing_hyperparameters"]
+    lines = [f"differing hyperparameters: {', '.join(diff) or '(none)'}"]
+    for r in c["runs"]:
+        m = ", ".join(f"{k}={v:.4g}" for k, v in r["metrics"].items())
+        hp = ", ".join(f"{k}={r['hyperparameters'].get(k)}" for k in diff)
+        star = " ★" if r["run_id"] == c["best_run_id"] else ""
+        lines.append(f"{r['run_id']:<40} {r['status']:<10} {m}  [{hp}]{star}")
+    emit(as_json, c, "\n".join(lines))

autoforge/config.py

@@ -0,0 +1,31 @@
+"""Paths and settings. Override the home directory with AUTOFORGE_HOME."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+DEFAULT_INDEX_URL = (
+    "https://raw.githubusercontent.com/Honguan/autoforge/packs-index/index.json"
+)
+
+
+def home() -> Path:
+    return Path(os.environ.get("AUTOFORGE_HOME", Path.home() / ".autoforge")).expanduser()
+
+
+def packs_dir() -> Path:
+    return home() / "packs"
+
+
+def lock_file() -> Path:
+    return home() / "packs.lock.json"
+
+
+def cache_dir() -> Path:
+    return home() / "cache"
+
+
+def index_url() -> str:
+    """URL or local file path of the pack index."""
+    return os.environ.get("AUTOFORGE_INDEX_URL", DEFAULT_INDEX_URL)

autoforge/data/profile.py

@@ -0,0 +1,99 @@
+"""Dataset profiling that respects the privacy guard.
+
+- summary access: schema and aggregate statistics only (no rows). Top values are
+  shown only for low-cardinality columns, which are categories, not identifiers.
+- full access: the same plus a few sample rows.
+"""
+
+from __future__ import annotations
+
+import csv
+from collections import Counter
+from pathlib import Path
+
+from autoforge.policy.privacy import Access
+
+MAX_ROWS = 200_000
+LOW_CARDINALITY = 20
+SAMPLE_ROWS = 5
+
+
+def _num(v: str) -> float | None:
+    try:
+        return float(v)
+    except ValueError:
+        return None
+
+
+def _profile_csv(path: Path, access: Access) -> dict:
+    with path.open(newline="", encoding="utf-8", errors="replace") as f:
+        reader = csv.reader(f)
+        header = next(reader, [])
+        cols = {h: {"values": Counter(), "missing": 0, "numeric": [], "non_numeric": 0} for h in header}
+        samples, n, truncated = [], 0, False
+        for row in reader:
+            if n >= MAX_ROWS:
+                truncated = True
+                break
+            n += 1
+            if len(samples) < SAMPLE_ROWS:
+                samples.append(dict(zip(header, row)))
+            for h, v in zip(header, row):
+                c = cols[h]
+                if v == "":
+                    c["missing"] += 1
+                    continue
+                if len(c["values"]) <= LOW_CARDINALITY or v in c["values"]:
+                    c["values"][v] += 1
+                x = _num(v)
+                if x is None:
+                    c["non_numeric"] += 1
+                else:
+                    c["numeric"].append(x)
+
+    columns = {}
+    for h, c in cols.items():
+        present = n - c["missing"]
+        is_numeric = present > 0 and c["non_numeric"] == 0
+        info: dict = {"dtype": "numeric" if is_numeric else "categorical", "missing": c["missing"]}
+        distinct = len(c["values"])
+        if distinct <= LOW_CARDINALITY:
+            info["distinct"] = distinct
+            info["top_values"] = dict(c["values"].most_common(5))
+        else:
+            info["distinct"] = f">{LOW_CARDINALITY}"
+        if is_numeric:
+            xs = c["numeric"]
+            info |= {"min": min(xs), "max": max(xs), "mean": round(sum(xs) / len(xs), 6)}
+        columns[h] = info
+
+    out = {"format": "csv", "rows": n, "rows_truncated_at": MAX_ROWS if truncated else None, "columns": columns}
+    if access is Access.full:
+        out["sample_rows"] = samples
+    return out
+
+
+def _profile_dir(path: Path) -> dict:
+    exts: Counter = Counter()
+    subdirs = sorted(p.name for p in path.iterdir() if p.is_dir())
+    files = 0
+    for p in path.rglob("*"):
+        if p.is_file():
+            files += 1
+            exts[p.suffix.lower() or "(none)"] += 1
+    return {
+        "format": "directory",
+        "files": files,
+        "extensions": dict(exts.most_common(10)),
+        "top_level_dirs": subdirs[:50],  # e.g. class folders in image datasets
+    }
+
+
+def profile(path: str | Path, access: Access) -> dict:
+    p = Path(path).expanduser().resolve()
+    base = {"path": str(p), "access": access.value}
+    if p.is_dir():
+        return base | _profile_dir(p)
+    if p.suffix.lower() == ".csv":
+        return base | _profile_csv(p, access)
+    return base | {"format": p.suffix.lstrip(".") or "unknown", "bytes": p.stat().st_size}

autoforge/data/registry.py

@@ -0,0 +1,47 @@
+"""Dataset sensitivity registry (~/.autoforge/datasets.json).
+
+Marking a directory covers every file inside it. Unmarked data is 'internal'.
+Only humans mark data (CLI); the MCP server exposes no way to change it, so a
+cloud AI cannot downgrade a dataset to get access.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from autoforge import config
+from autoforge.schemas.core import Sensitivity
+
+DEFAULT = Sensitivity.internal
+
+
+def _file() -> Path:
+    return config.home() / "datasets.json"
+
+
+def _load() -> dict[str, str]:
+    f = _file()
+    return json.loads(f.read_text(encoding="utf-8")) if f.is_file() else {}
+
+
+def mark(path: str | Path, sensitivity: Sensitivity) -> str:
+    key = str(Path(path).expanduser().resolve())
+    data = _load() | {key: sensitivity.value}
+    _file().parent.mkdir(parents=True, exist_ok=True)
+    _file().write_text(json.dumps(dict(sorted(data.items())), indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return key
+
+
+def sensitivity_of(path: str | Path) -> Sensitivity:
+    """The closest marked ancestor (or the path itself) decides; default 'internal'."""
+    data = _load()
+    p = Path(path).expanduser().resolve()
+    for candidate in (p, *p.parents):
+        if str(candidate) in data:
+            return Sensitivity(data[str(candidate)])
+    return DEFAULT
+
+
+def all_marks() -> dict[str, str]:
+    return _load()

autoforge/engine/context.py

@@ -0,0 +1,125 @@
+"""TrainContext: the object every trainer receives.
+
+Trainer contract (v1):
+    def train(ctx: TrainContext) -> dict:
+        ...
+        return {"metrics": {"accuracy": 0.9}, "model_path": "..."}
+
+Trainers should call `ctx.log_metric(...)` as they go and `ctx.check_stop()`
+between expensive steps so a stop request can end the run cleanly.
+Pause works without trainer cooperation (the process is suspended).
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+
+class RunStopped(Exception):
+    """Raised by ctx.check_stop() when a stop was requested."""
+
+
+class TrainContext:
+    def __init__(
+        self,
+        run_dir: Path,
+        data_path: str,
+        target: str | None,
+        task_type: str,
+        hyperparameters: dict[str, Any],
+        device: str,
+        seed: int = 0,
+        resume_from: str | None = None,
+    ):
+        self.run_dir = Path(run_dir)
+        self.data_path = data_path
+        self.target = target
+        self.task_type = task_type
+        self.hyperparameters = hyperparameters
+        self.device = device
+        self.seed = seed
+        self.output_dir = self.run_dir / "output"
+        self.checkpoint_dir = self.run_dir / "checkpoints"
+        self.resume_from = Path(resume_from) if resume_from else None
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
+        self._events = self.run_dir / "events.jsonl"
+        self._control = self.run_dir / "control.json"
+        self._last_check = 0.0
+        self._stop = False
+        self._adjust_seq = 0
+
+    # ---- metrics / logging ----
+
+    def log_metric(self, name: str, value: float, step: int | None = None) -> None:
+        event = {"t": time.time(), "type": "metric", "name": name, "value": float(value), "step": step}
+        with self._events.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(event) + "\n")
+
+    def log(self, message: str) -> None:
+        event = {"t": time.time(), "type": "log", "message": message}
+        with self._events.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(event, ensure_ascii=False) + "\n")
+        print(message, flush=True)
+
+    # ---- control ----
+
+    def should_stop(self) -> bool:
+        """True once a stop was requested. Reads the control file at most every 0.5 s."""
+        now = time.monotonic()
+        if not self._stop and now - self._last_check >= 0.5:
+            self._last_check = now
+            try:
+                self._stop = bool(json.loads(self._control.read_text()).get("stop_requested"))
+            except (OSError, ValueError):
+                pass
+        return self._stop
+
+    def check_updates(self) -> dict[str, Any]:
+        """Hyperparameter changes sent with `run adjust` since the last call (already applied to
+        self.hyperparameters). Trainers that support live changes call this between steps."""
+        try:
+            control = json.loads(self._control.read_text())
+        except (OSError, ValueError):
+            return {}
+        seq = int(control.get("adjust_seq", 0))
+        if seq <= self._adjust_seq:
+            return {}
+        self._adjust_seq = seq
+        changes = dict(control.get("overrides") or {})
+        changed = {k: v for k, v in changes.items() if self.hyperparameters.get(k) != v}
+        self.hyperparameters.update(changed)
+        if changed:
+            self.log(f"hyperparameters adjusted: {changed}")
+        return changed
+
+    def check_stop(self) -> None:
+        if self.should_stop():
+            raise RunStopped()
+
+    def latest_checkpoint(self) -> Path | None:
+        """Newest file in resume_from (if retrying) or this run's checkpoint dir."""
+        for d in (self.resume_from, self.checkpoint_dir):
+            if d and d.is_dir():
+                files = sorted((p for p in d.iterdir() if p.is_file()), key=lambda p: p.stat().st_mtime)
+                if files:
+                    return files[-1]
+        return None
+
+    # ---- compatibility with the provisional dict contract ----
+
+    def __getitem__(self, key: str) -> Any:
+        if key in ("data_path", "target", "task_type", "hyperparameters", "seed", "device"):
+            return getattr(self, key)
+        if key == "output_dir":
+            return str(self.output_dir)
+        raise KeyError(key)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        try:
+            return self[key]
+        except KeyError:
+            return default

autoforge/engine/diagnose.py

@@ -0,0 +1,134 @@
+"""Deterministic run diagnosis (P5). Gives any AI, including small local models,
+concrete findings to act on instead of raw logs.
+
+Each finding: {severity: error|warning|info, code, message, suggestion}.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+
+from autoforge.engine import runner
+from autoforge.engine.store import RunStore
+from autoforge.packs import installer
+from autoforge.packs.errors import PackError
+from autoforge.schemas.core import RunStatus
+
+LOG_RULES: list[tuple[str, str, str, str]] = [
+    # (regex, code, message, suggestion)
+    (r"No module named '([\w.]+)'", "MISSING_PACKAGE", "Python package '{0}' is not installed",
+     "pip install {0} (check pack_info missing_dependencies)"),
+    (r"CUDA out of memory|OutOfMemoryError|CUBLAS_STATUS_ALLOC_FAILED", "GPU_OOM", "GPU ran out of memory",
+     "lower batch_size, use a smaller model, enable gradient accumulation or 4-bit/QLoRA"),
+    (r"MemoryError|Killed", "RAM_OOM", "the process ran out of RAM",
+     "use fewer rows or features, lower batch_size, or drop high-cardinality columns"),
+    (r"least populated class in y has only 1 member|The test_size .* should be greater or equal to the number of classes",
+     "RARE_CLASS", "a class has too few examples to split", "merge or drop rare classes, or collect more data"),
+    (r"could not convert string to float: '([^']*)'", "NON_NUMERIC", "a non-numeric value '{0}' reached a numeric step",
+     "encode categorical columns or clean the column containing that value"),
+    (r"KeyError: '([^']+)'", "MISSING_COLUMN", "column '{0}' was not found",
+     "check task.target and the CSV header (data_profile)"),
+    (r"FileNotFoundError: .*?'([^']+)'", "FILE_NOT_FOUND", "file '{0}' was not found", "check data_path"),
+]
+LOSS_LIKE = ("loss", "rmse", "mse", "mae", "error", "perplexity")
+
+
+def _is_loss(name: str) -> bool:
+    return any(h in name.lower() for h in LOSS_LIKE)
+
+
+def _finding(severity: str, code: str, message: str, suggestion: str = "") -> dict:
+    return {"severity": severity, "code": code, "message": message, "suggestion": suggestion}
+
+
+def _series(store: RunStore) -> dict[str, list[float]]:
+    out: dict[str, list[float]] = {}
+    for e in store.events():
+        if e.get("type") == "metric" and e.get("step") is not None:
+            out.setdefault(e["name"], []).append(float(e["value"]))
+    return out
+
+
+def _log_findings(store: RunStore) -> list[dict]:
+    text = "\n".join(store.logs_tail(300))
+    found = []
+    for pattern, code, msg, sug in LOG_RULES:
+        m = re.search(pattern, text)
+        if m:
+            groups = m.groups() or ("",)
+            found.append(_finding("error", code, msg.format(*groups), sug.format(*groups)))
+    return found
+
+
+def _curve_findings(series: dict[str, list[float]], tunable: set[str]) -> list[dict]:
+    found = []
+    lr = next((k for k in ("learning_rate", "lr") if k in tunable), "learning_rate")
+    for name, xs in series.items():
+        if any(not math.isfinite(x) for x in xs):
+            found.append(_finding("error", "DIVERGED", f"'{name}' became NaN/inf",
+                                  f"lower {lr} (e.g. ÷10), add gradient clipping, check input scaling"))
+            continue
+        if len(xs) < 4 or not _is_loss(name):
+            continue
+        best_i = min(range(len(xs)), key=xs.__getitem__)
+        if xs[-1] > xs[best_i] * 1.2 and best_i < len(xs) - 2 and not name.lower().startswith(("val", "valid", "eval", "test")):
+            found.append(_finding("warning", "LOSS_RISING", f"'{name}' rose {xs[-1] / xs[best_i]:.2f}× above its best",
+                                  f"lower {lr}; the run may be unstable"))
+        half = xs[len(xs) // 2:]
+        if len(half) >= 3 and half[0] != 0 and abs(half[0] - half[-1]) / abs(half[0]) < 0.01:
+            found.append(_finding("info", "PLATEAU", f"'{name}' changed < 1% over the last half of training",
+                                  f"stop earlier to save time, or raise {lr} / model capacity if the score is too low"))
+    # overfitting: training loss keeps falling while validation loss rises
+    for name, val in series.items():
+        lname = name.lower()
+        if not lname.startswith(("val_", "valid_", "eval_")) or not _is_loss(name) or len(val) < 4:
+            continue
+        base = name.split("_", 1)[1]
+        train = series.get(f"train_{base}") or series.get(base)
+        if not train or len(train) < 4:
+            continue
+        best_i = min(range(len(val)), key=val.__getitem__)
+        if best_i < len(val) - 2 and val[-1] > val[best_i] * 1.05 and train[-1] < train[len(train) // 2]:
+            reg = [k for k in ("dropout", "weight_decay", "min_samples_leaf", "max_depth", "C") if k in tunable]
+            found.append(_finding("warning", "OVERFITTING",
+                                  f"'{name}' was best at step {best_i} and then rose while training loss kept falling",
+                                  "stop at the best step (early stopping)" + (f"; regularize via {', '.join(reg)}" if reg else "")))
+    return found
+
+
+def diagnose(store: RunStore) -> dict:
+    view = runner.status(store)
+    rec = store.record()
+    tunable: set[str] = set(rec.plan.hyperparameters)
+    if rec.plan.pack:
+        try:
+            _, pack = installer.get_installed(rec.plan.pack)
+            recipe = pack.recipe(rec.plan.recipe or "")
+            if recipe:
+                tunable |= set(recipe.search_space)
+        except PackError:
+            pass
+
+    findings: list[dict] = []
+    if view.status is RunStatus.failed:
+        findings += _log_findings(store) or [_finding(
+            "error", "UNKNOWN_FAILURE", view.error or "run failed", "read run_logs for the traceback")]
+    elif view.status is RunStatus.stopped:
+        findings.append(_finding("info", "STOPPED", "the run was stopped before finishing",
+                                 "run_retry resumes from checkpoints if the trainer saves them"))
+    findings += _curve_findings(_series(store), tunable)
+
+    for name, value in view.metrics.items():
+        if name.lower() in ("accuracy", "f1", "f1_macro", "auc", "roc_auc", "r2") and value >= 0.999:
+            findings.append(_finding("warning", "POSSIBLE_LEAKAGE", f"{name} = {value:.4f} is suspiciously perfect",
+                                     "check whether a feature leaks the target (e.g. an ID or post-outcome column)"))
+
+    status_text = view.status.value
+    if not findings:
+        summary = f"{status_text}; no problems detected"
+    else:
+        worst = next((s for s in ("error", "warning", "info") if any(f["severity"] == s for f in findings)), "info")
+        summary = f"{status_text}; {len(findings)} finding(s), worst: {worst}"
+    return {"run_id": store.run_id, "status": status_text, "metrics": view.metrics,
+            "tunable_hyperparameters": sorted(tunable), "findings": findings, "summary": summary}