agentpace 0.9.2__py3-none-any.whl

agentpace/report.py

@@ -0,0 +1,502 @@
+"""Turn backtest numbers into `docs/backtest.md`, its JSON twin and its charts."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from agentpace import backtest, charts
+from agentpace import estimate as est
+from agentpace.schema import Row
+
+_NOTE = "<!-- Generated by `agentpace backtest`. Do not edit by hand. -->"
+_MIN_CHART_ROWS = 10
+
+
+def _rounded(value: Any) -> Any:
+    """Four significant figures, so results compare equal across platforms."""
+    if isinstance(value, float):
+        return float(f"{value:.4g}")
+    if isinstance(value, dict):
+        return {key: _rounded(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [_rounded(item) for item in value]
+    return value
+
+
+def to_json(report: dict[str, Any]) -> str:
+    return json.dumps(_rounded(report), indent=2) + "\n"
+
+
+def _metric_rows(results: dict[str, Any], names: Sequence[str], weighting: str) -> list[str]:
+    lines = [
+        "| Method | MAPE | MdAPE | MALE | P50 cov. | P80 cov. | P90 cov. | P90/P50 | Pinball |",
+        "|---|---:|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for name in names:
+        m = results[name][weighting]
+        lines.append(
+            f"| {name} | {m['mape']:.0f}% | {m['mdape']:.0f}% | {m['male']:.2f} | "
+            f"{m['coverage50']:.1f}% | {m['coverage80']:.1f}% | {m['coverage90']:.1f}% | "
+            f"{m['width']:.1f}x | {m['pinball']:.0f} |"
+        )
+    return lines
+
+
+def ci(value: float, interval: Sequence[float], digits: int = 0, unit: str = "") -> str:
+    """A number with its 95% interval, such as "80% (CI 70% to 91%)"."""
+    low, high = interval
+    return f"{value:.{digits}f}{unit} (CI {low:.{digits}f}{unit} to {high:.{digits}f}{unit})"
+
+
+def _gap(interval: Sequence[float]) -> str:
+    low, high = interval
+    return f"{low:+.0f} to {high:+.0f}"
+
+
+def _interval_lines(
+    results: dict[str, Any], winner: str, found: dict[str, Any], weighting: str = "session"
+) -> list[str]:
+    m = results[winner][weighting]
+    lines = [
+        f"`{winner}` with 95% session-bootstrap intervals: P50 coverage "
+        f"{ci(m['coverage50'], found['coverage50'], 1, '%')}, P80 coverage "
+        f"{ci(m['coverage80'], found['coverage80'], 1, '%')}, P90 coverage "
+        f"{ci(m['coverage90'], found['coverage90'], 1, '%')}, pinball "
+        f"{ci(m['pinball'], found['pinball'])}.",
+        "",
+        f"| Baseline | Pinball | Baseline minus `{winner}` (95% interval) |",
+        "|---|---:|---|",
+    ]
+    for name, gap in found["rival_minus_ours"].items():
+        lines.append(f"| {name} | {results[name][weighting]['pinball']:.0f} | {_gap(gap)} |")
+    lines += [
+        "",
+        "An interval wholly above zero means the baseline is worse at the 95% level.",
+        "",
+    ]
+    return lines
+
+
+def _week_section(weeks: Sequence[dict[str, Any]], winner: str, last: str) -> list[str]:
+    lines = [
+        "## Time split, week by week",
+        "",
+        f"The pooled numbers above average over weeks that miss in both directions, so here is "
+        f"`{winner}` fold by fold. Weeks up to {last} chose the method. Later weeks are the "
+        "locked holdout. Pinball is session-weighted.",
+        "",
+        "| Week | Period | Sessions | Runs | P80 cov., per session | P80 cov., per run | "
+        "P90/P50 | Pinball | Global-median pinball |",
+        "|---|---|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for w in weeks:
+        lines.append(
+            f"| {w['week']} | {w['period']} | {w['sessions']} | {w['rows']:,} | "
+            f"{w['coverage80_session']:.0f}% | {w['coverage80_row']:.0f}% | {w['width']:.1f}x | "
+            f"{w['pinball']:.0f} | {w['median_pinball']:.0f} |"
+        )
+    lines.append("")
+    return lines
+
+
+def _locked_section(locked: dict[str, Any], winner: str, last: str) -> list[str]:
+    lines = [
+        "## Locked holdout",
+        "",
+        f"Weeks after {last} never take part in selection. They score the shipped `{winner}` on "
+        f"data it was not chosen on. Results print once at least {locked['min_sessions']} "
+        f"held-out sessions have {est.MIN_SESSION_RUNS} or more completed runs.",
+        "",
+    ]
+    if locked["status"] != "ok":
+        status = locked["status"]
+        return [*lines, f"**{status[0].upper()}{status[1:]}.**", "", *_locked_harness_lines(locked)]
+    results = locked["tracks"]["pre-run"]
+    names = [winner, *backtest.BASELINES]
+    lines += [f"Weeks {', '.join(locked['folds'])}. Each session counts equally:", ""]
+    lines += _metric_rows(results, names, "session")
+    lines += ["", "Each run counts equally:", ""]
+    lines += _metric_rows(results, names, "row")
+    lines += [""]
+    lines += _interval_lines(results, winner, locked["intervals"])
+    lines += [
+        f"Whole sessions, {locked['qualifying_sessions']} of them:",
+        "",
+        *_plan_rows(locked["plans"]),
+        "",
+        *_locked_harness_lines(locked),
+    ]
+    return lines
+
+
+def _method_names() -> list[str]:
+    return [method.name for method in est.METHODS] + list(backtest.BASELINES)
+
+
+def _split_section(title: str, split: dict[str, Any], track: str, intro: str) -> list[str]:
+    if split["status"] != "ok":
+        return [f"## {title}", "", f"**{split['status'].capitalize()}.**", ""]
+    results = split["tracks"][track]
+    lines = [f"## {title}", "", intro, ""]
+    if all(results[name]["session"] == results[name]["row"] for name in _method_names()):
+        lines += [
+            "Each session counts equally, and each run counts equally: every held-out session "
+            "here has the same number of runs, so both weightings give the same table.",
+            "",
+        ]
+        lines += _metric_rows(results, _method_names(), "session")
+        lines.append("")
+        return lines
+    lines += ["Each session counts equally:", ""]
+    lines += _metric_rows(results, _method_names(), "session")
+    lines += ["", "Each run counts equally:", ""]
+    lines += _metric_rows(results, _method_names(), "row")
+    lines.append("")
+    return lines
+
+
+def _selection_section(selection: dict[str, Any]) -> list[str]:
+    lines = [
+        "## Which method ships",
+        "",
+        f"The rule was fixed before any result existed: lowest {selection['primary_metric']}, on "
+        "the time split, with ties inside the 95% session-bootstrap interval going to the "
+        f"simpler candidate. Selection uses weeks up to {selection['last_week']} and never re-runs "
+        "on later ones.",
+        "",
+        *_population_lines(selection),
+        "| Candidate, simplest first | Loss | Difference from lowest (95% interval) |",
+        "|---|---:|---|",
+    ]
+    for name, loss in selection["loss"].items():
+        low, high = selection["difference_from_best"][name]
+        gap = "lowest" if name == selection["lowest_loss"] else f"{low:+.0f} to {high:+.0f}"
+        lines.append(f"| {name} | {loss:.0f} | {gap} |")
+    lines += [
+        "",
+        f"**Winner: `{selection['winner']}`.** It is the default for `agentpace estimate`.",
+        "",
+    ]
+    return lines
+
+
+def _population_lines(selection: dict[str, Any]) -> list[str]:
+    # Tied to `backtest.SELECTION_HARNESS`: if that constant ever changes, so must this text.
+    harness = selection.get("harness", "all")
+    if harness == "all":
+        return []
+    return [
+        f"Selection, the time-split tables and the whole-session test score `{harness}` sessions "
+        "only, the population the rule was frozen on. Rows from every harness train the "
+        "estimator. This scope was registered on 2026-11-07, after a local run with Codex rows "
+        "pooled in had been seen.",
+        "",
+    ]
+
+
+def _pooled_section(pooled: dict[str, Any]) -> list[str]:
+    if pooled["status"] != "ok":
+        return []
+    lines = [
+        "## Pooled, for the record",
+        "",
+        "The same rule over every harness at once. It is never shipped. It is printed so you can "
+        "judge the choice above. Pinball is session-weighted, so a harness with many short "
+        "sessions carries most of this vote.",
+        "",
+        "| Candidate, simplest first | Loss | Difference from lowest (95% interval) |",
+        "|---|---:|---|",
+    ]
+    for name, loss in pooled["loss"].items():
+        low, high = pooled["difference_from_best"][name]
+        gap = "lowest" if name == pooled["lowest_loss"] else f"{low:+.0f} to {high:+.0f}"
+        lines.append(f"| {name} | {loss:.0f} | {gap} |")
+    lines += ["", f"Pooled winner: `{pooled['winner']}`. Baselines on the same sessions:", ""]
+    lines += ["| Baseline | Pinball |", "|---|---:|"]
+    for name, loss in pooled["baselines"].items():
+        lines.append(f"| {name} | {loss:.0f} |")
+    return [*lines, ""]
+
+
+def _locked_harness_lines(locked: dict[str, Any]) -> list[str]:
+    found = locked.get("by_harness")
+    if not found:
+        return []
+    lines = [
+        "Each harness at task level, every candidate, each session counting equally. This is "
+        "where a harness that selection does not score is tested, on weeks no method was "
+        "chosen on.",
+        "",
+        "| Harness | Method | Sessions | Runs | Pinball | P80 cov. | P90/P50 |",
+        "|---|---|---:|---:|---:|---:|---:|",
+    ]
+    for harness, entry in found.items():
+        for name, m in entry["methods"].items():
+            lines.append(
+                f"| {harness} | {name} | {entry['sessions']} | {entry['rows']:,} | "
+                f"{m['pinball']:.0f} | {m['coverage80']:.1f}% | {m['width']:.1f}x |"
+            )
+    return [*lines, ""]
+
+
+def _plan_section(plans: dict[str, Any]) -> list[str]:
+    lines = [
+        "## Whole sessions (oracle-structure)",
+        "",
+        f"Each of {plans['sessions']} held-out sessions with at least 5 completed runs is treated "
+        "as a plan. This is the only direct test of plan totals. Real dependencies are not "
+        "recorded, so the critical path falls back to the longest run, and the observed peak "
+        "concurrency stands in for the agent cap, even though that peak is itself an outcome. "
+        "It tests the throughput and session terms, not dependency planning.",
+        "",
+        *_plan_rows(plans["results"]),
+        "",
+    ]
+    found = plans["intervals"]
+    ours = plans["results"]["agentpace"]
+    lines += [
+        "With 95% bootstrap intervals over sessions: P50 coverage "
+        f"{ci(ours['coverage50'], found['coverage50'], unit='%')}, P80 coverage "
+        f"{ci(ours['coverage80'], found['coverage80'], unit='%')}, P90 coverage "
+        f"{ci(ours['coverage90'], found['coverage90'], unit='%')}, pinball "
+        f"{ci(ours['pinball'], found['pinball'])}. Sum-of-medians pinball minus agentpace: "
+        f"{_gap(found['rival_minus_ours']['sum-of-medians'])}.",
+        "",
+    ]
+    return lines
+
+
+def _plan_rows(results: dict[str, Any]) -> list[str]:
+    lines = [
+        "| Method | MAPE | MdAPE | MALE | P50 cov. | P80 cov. | P90 cov. | P90/P50 | Pinball |",
+        "|---|---:|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for name, m in results.items():
+        lines.append(
+            f"| {name} | {m['mape']:.0f}% | {m['mdape']:.0f}% | {m['male']:.2f} | "
+            f"{m['coverage50']:.0f}% | {m['coverage80']:.0f}% | {m['coverage90']:.0f}% | "
+            f"{m['width']:.1f}x | {m['pinball']:.0f} |"
+        )
+    return lines
+
+
+def _project_lines(project: dict[str, Any], winner: str) -> list[str]:
+    """Each held-out project on its own, by rank, so one project's vote is visible."""
+    if "by_project" not in project:
+        return []
+    lines = [
+        "Each project on its own, most held-out sessions first. Projects are numbered by rank, "
+        "not named, so the numbers change as data is added.",
+        "",
+        f"| Project | Harness | Test sessions | Runs | `{winner}` pinball | "
+        "global-median pinball | P80 coverage, per session | P80 coverage, per run | P90/P50 |",
+        "|---:|---|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for rank, m in enumerate(project["by_project"], start=1):
+        lines.append(
+            f"| {rank} | {m['harness']} | {m['sessions']} | {m['rows']:,} | {m['pinball']:.0f} | "
+            f"{m['median_pinball']:.0f} | {m['coverage80_session']:.1f}% | "
+            f"{m['coverage80_row']:.1f}% | {m['width']:.1f}x |"
+        )
+    if "without_largest" not in project:
+        return [*lines, ""]
+    largest = project["without_largest"]
+    losses = ", ".join(f"{name} {value:.0f}" for name, value in largest["loss"].items())
+    if largest["lowest"] == largest["lowest_with_it"]:
+        verdict = f"The lowest candidate is still `{largest['lowest']}`."
+    else:
+        verdict = (
+            f"The lowest candidate changes from `{largest['lowest_with_it']}` to "
+            f"`{largest['lowest']}`."
+        )
+    lines += [
+        "",
+        f"Without project {largest['project']} ({largest['sessions']} of "
+        f"{largest['of_sessions']} test sessions), session-weighted pinball: {losses}. "
+        f"{verdict} This split does not choose the method.",
+        "",
+    ]
+    return lines
+
+
+def _harness_section(harness: dict[str, Any], split: dict[str, Any], winner: str) -> list[str]:
+    lines = [
+        "## By harness",
+        "",
+        "The estimator does not use the harness. Each harness is shown on its own, from the "
+        "same selection-fold forecasts, before anyone pools them.",
+        "",
+        "| Harness | Runs | Completed | Sessions |",
+        "|---|---:|---:|---:|",
+    ]
+    for name, counts in harness["dataset"].items():
+        lines.append(
+            f"| {name} | {counts['rows']:,} | {counts['completed']:,} | {counts['sessions']} |"
+        )
+    lines += [
+        "",
+        f"| Harness | Test sessions | `{winner}` pinball | global-median pinball | "
+        "P80 coverage, per session | P80 coverage, per run | P90/P50 |",
+        "|---|---:|---:|---:|---:|---:|---:|",
+    ]
+    for name, m in harness["selection_folds"].items():
+        lines.append(
+            f"| {name} | {m['sessions']} | {m['pinball']:.0f} | {m['median_pinball']:.0f} | "
+            f"{m['coverage80_session']:.1f}% | {m['coverage80_row']:.1f}% | {m['width']:.1f}x |"
+        )
+    if split["status"] != "ok":
+        lines += ["", f"Harness split: **{split['status']}.**"]
+    return [*lines, ""]
+
+
+def _loss_section(losses: Sequence[dict[str, Any]], winner: str) -> list[str]:
+    lines = [
+        "## Where agentpace loses",
+        "",
+        f"This section is generated. It lists every split, track, weighting and metric where a "
+        f"baseline beat `{winner}`. Lower is better, except P80 coverage, where nearer 80% wins.",
+        "",
+    ]
+    if not losses:
+        return [*lines, "No baseline won on any metric.", ""]
+    lines += [
+        "| Split | Track | Weighting | Metric | Baseline | Baseline value | agentpace value |",
+        "|---|---|---|---|---|---:|---:|",
+    ]
+    for item in losses:
+        lines.append(
+            f"| {item['split']} | {item['track']} | {item['weighting']} | {item['metric']} | "
+            f"{item['baseline']} | {item['baseline_value']:.2f} | {item['agentpace_value']:.2f} |"
+        )
+    lines.append("")
+    return lines
+
+
+def markdown(report: dict[str, Any]) -> str:
+    data = report["dataset"]
+    weeks = data["weeks"]
+    lines = [
+        _NOTE,
+        "# Backtest",
+        "",
+        f"From {data['rows']:,} runs ({data['completed']:,} completed) in {data['sessions']} "
+        f"sessions across {data['projects']} projects, {weeks[0]} to {weeks[-1]}, from "
+        f"{data['contributors']} contributor{'' if data['contributors'] == 1 else 's'}. "
+        f"The largest project holds "
+        f"{data['top_project_share']:.0f}% of runs and the three largest sessions hold "
+        f"{data['top_three_session_share']:.0f}%.",
+        "",
+        f"Reproduce with `agentpace backtest` (seed {report['seed']}). Forecasts use only "
+        "`model`, `task_kind` and `risk` unless a track says otherwise.",
+        "",
+        "![P80 coverage by method](assets/backtest-coverage.svg)",
+        "",
+        "Reading the columns: MAPE is the mean percentage error of the P50, which heavy tails "
+        "inflate. MdAPE is its median. MALE is mean absolute log error. Coverage is how often "
+        "the actual run landed at or under that forecast. P90/P50 is the median width of the "
+        "range. Pinball is in seconds, summed over 0.5, 0.8 and 0.9.",
+        "",
+    ]
+    lines += _selection_section(report["selection"])
+    stored = _rounded(report)
+    lines += _pooled_section(stored.get("pooled_selection", {"status": "absent"}))
+    splits = report["splits"]
+    winner = report["selection"]["winner"]
+    last = report["selection"]["last_week"]
+    # The README quotes the stored JSON, which is rounded. Sections added after 0.1.0 render
+    # from the same rounded values so the two documents never disagree in the last digit.
+    lines += _split_section(
+        "Time split, pre-run",
+        splits["time"],
+        "pre-run",
+        f"Train on earlier weeks, test on {', '.join(splits['time']['folds'])}. This is the "
+        "honest forecast and the headline.",
+    )
+    time = stored["splits"]["time"]
+    lines += _interval_lines(time["tracks"]["pre-run"], winner, time["intervals"])
+    lines += _week_section(time["by_week"], winner, last)
+    lines += _split_section(
+        "Time split, oracle-size",
+        splits["time"],
+        "oracle-size",
+        "The same split with stored `size` added. Size is measured after a run, so this is an "
+        "upper bound, not a forecast anyone could make.",
+    )
+    if splits["project"]["status"] == "ok":
+        lines += _split_section(
+            "Project split",
+            splits["project"],
+            "pre-run",
+            f"Leave one project out, over {splits['project']['folds']} projects. This tests "
+            "generalizing to a new codebase, not to a new person.",
+        )
+        lines += _project_lines(stored["splits"]["project"], winner)
+    else:
+        lines += _split_section("Project split", splits["project"], "pre-run", "")
+    if splits["contributor"]["status"] == "ok":
+        lines += _split_section(
+            "Contributor split", splits["contributor"], "pre-run", "Leave one contributor out."
+        )
+    else:
+        lines += _split_section("Contributor split", splits["contributor"], "pre-run", "")
+    if splits["harness"]["status"] == "ok":
+        lines += _split_section(
+            "Harness split", splits["harness"], "pre-run", "Leave one harness out."
+        )
+    lines += _harness_section(stored["harness"], splits["harness"], winner)
+    lines += _plan_section(stored["plans"])
+    lines += _locked_section(stored["locked"], winner, last)
+    lines += _loss_section(report["losses"], winner)
+    baselines = (
+        "## Baselines",
+        "",
+        "- `human-guess`: a hand-written table of hours per kind. An assumption, not a "
+        "measurement.",
+        "- `fixed-rounds`: 3 minutes per round, with rounds from a hand-written table per kind.",
+        "- `rounds-oracle`: 3 minutes times the actual `tool_calls`. It uses the answer, so no "
+        "forecast could do this. It is shown anyway.",
+        "- `global-median`: the training median, for every run.",
+        "",
+        "Baselines give one number, used for every quantile, so their P90/P50 is always 1.0x.",
+    )
+    return "\n".join([*lines, *baselines]) + "\n"
+
+
+def run_range_chart(rows: Sequence[Row]) -> str:
+    completed = [row for row in rows if row["outcome"] == "completed"]
+    bars = []
+    for kind in sorted({row["task_kind"] for row in completed}):
+        values = sorted(row["duration_s"] for row in completed if row["task_kind"] == kind)
+        if len(values) >= _MIN_CHART_ROWS:
+            p50, p80, p90 = (est.quantile(values, q) for q in est.QUANTILES)
+            bars.append(charts.RangeBar(kind, len(values), p50, p80, p90))
+    bars.sort(key=lambda bar: bar.p50)
+    return charts.range_chart("How long agent runs took, by task kind", bars)
+
+
+def coverage_chart(report: dict[str, Any]) -> str:
+    results = report["splits"]["time"]["tracks"]["pre-run"]
+    points = [(m.name, results[m.name]["session"]["coverage80"], False) for m in est.METHODS]
+    points += [(name, results[name]["session"]["coverage80"], True) for name in backtest.BASELINES]
+    return charts.coverage_chart("P80 coverage on the time split, per session", points, 80.0)
+
+
+def write(report: dict[str, Any], rows: Sequence[Row], out: Path) -> None:
+    (out / "assets").mkdir(parents=True, exist_ok=True)
+    files = {
+        out / "backtest.md": markdown(report),
+        out / "backtest.json": to_json(report),
+        out / "assets" / "backtest-coverage.svg": coverage_chart(report),
+        out / "assets" / "run-ranges.svg": run_range_chart(rows),
+    }
+    for path, text in files.items():
+        path.write_text(text, encoding="utf-8", newline="\n")
+
+
+def is_current(report: dict[str, Any], out: Path) -> bool:
+    """Compare rounded numbers rather than chart bytes, so this holds on every platform."""
+    stored = out / "backtest.json"
+    return stored.exists() and stored.read_text(encoding="utf-8") == to_json(report)

agentpace/schema.py

@@ -0,0 +1,131 @@
+"""Row validation against `schema/run.schema.json`, without third-party packages.
+
+This implements only the JSON Schema keywords that file uses. An unknown keyword raises instead
+of being ignored, so the schema cannot quietly gain a rule that nothing enforces.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Iterable
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from agentpace import paths
+
+Row = dict[str, Any]
+
+_ANNOTATIONS = {"description", "title", "$schema", "$id"}
+_KNOWN = _ANNOTATIONS | {
+    "type",
+    "enum",
+    "pattern",
+    "minimum",
+    "required",
+    "properties",
+    "additionalProperties",
+}
+
+
+@lru_cache(maxsize=4)
+def load_schema_file(path: Path) -> dict[str, Any]:
+    schema: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
+    for node in [schema, *schema["properties"].values()]:
+        unknown = set(node) - _KNOWN
+        if unknown:
+            raise ValueError(f"unsupported schema keywords: {sorted(unknown)}")
+    return schema
+
+
+def load_schema() -> dict[str, Any]:
+    return load_schema_file(paths.schema_file())
+
+
+def fields() -> tuple[str, ...]:
+    """Row keys in canonical order."""
+    return tuple(load_schema()["properties"])
+
+
+def enum(field: str) -> tuple[str, ...]:
+    return tuple(load_schema()["properties"][field]["enum"])
+
+
+def _check_value(name: str, value: Any, rule: dict[str, Any]) -> str | None:
+    if "enum" in rule and value not in rule["enum"]:
+        return f"{name}: not an allowed value"
+    expected = rule.get("type")
+    if expected == "integer" and (not isinstance(value, int) or isinstance(value, bool)):
+        return f"{name}: must be an integer"
+    if expected == "string" and not isinstance(value, str):
+        return f"{name}: must be a string"
+    if "minimum" in rule and isinstance(value, int) and value < rule["minimum"]:
+        return f"{name}: below minimum {rule['minimum']}"
+    if "pattern" in rule and isinstance(value, str) and not re.fullmatch(rule["pattern"], value):
+        return f"{name}: does not match the required pattern"
+    return None
+
+
+def check_row(row: Any) -> list[str]:
+    """Return every schema violation. Messages name fields, never echo values."""
+    return check_against(load_schema(), row)
+
+
+def check_against(schema: dict[str, Any], row: Any) -> list[str]:
+    """`check_row` for any schema this module can read."""
+    if not isinstance(row, dict):
+        return ["row is not a JSON object"]
+    properties = schema["properties"]
+    errors = [f"{name}: missing" for name in schema["required"] if name not in row]
+    errors += [f"unknown field #{i}" for i, name in enumerate(row) if name not in properties]
+    for name, rule in properties.items():
+        if name in row:
+            problem = _check_value(name, row[name], rule)
+            if problem:
+                errors.append(problem)
+    return errors
+
+
+def canonical(row: Row) -> str:
+    """One row as one line of JSON, in schema field order."""
+    ordered = {name: row[name] for name in fields() if name in row}
+    return json.dumps(ordered, separators=(",", ":"), ensure_ascii=True)
+
+
+def dedupe_key(row: Row) -> tuple[str, str, int, int]:
+    """Identifies a run, not its labels, so a relabeled re-share is still a duplicate."""
+    return (row["contributor"], row["batch"], row["offset_s"], row["duration_s"])
+
+
+def read_rows(path: Path) -> list[Row]:
+    rows: list[Row] = []
+    with path.open(encoding="utf-8") as handle:
+        for line in handle:
+            if line.strip():
+                rows.append(json.loads(line))
+    return rows
+
+
+def read_dataset(sources: Iterable[Path]) -> list[Row]:
+    """Load rows from files or directories of `.jsonl`, keeping one row per run."""
+    merged: dict[tuple[str, str, int, int], Row] = {}
+    for source in sources:
+        files = sorted(source.glob("*.jsonl")) if source.is_dir() else [source]
+        for file in files:
+            for row in read_rows(file):
+                merged[dedupe_key(row)] = row
+    return [merged[key] for key in sorted(merged)]
+
+
+def write_rows(path: Path, rows: Iterable[Row]) -> int:
+    """Write atomically. A hook can be killed at its timeout, and a half-written store would make
+    every later read fail."""
+    lines = [canonical(row) for row in rows]
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + ".tmp")
+    with temporary.open("w", encoding="utf-8", newline="\n") as handle:
+        handle.writelines(line + "\n" for line in lines)
+    os.replace(temporary, path)
+    return len(lines)