playcal 0.1.0__py3-none-any.whl

playcal/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

playcal/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

playcal/affected.py

@@ -0,0 +1,115 @@
+"""Which Unity test assemblies a change can break, from the project's .asmdef graph.
+
+A changed .cs file belongs to the assembly of the nearest .asmdef (or .asmref) above it, or to
+Assembly-CSharp / Assembly-CSharp-Editor when there is none. Every assembly that references a changed
+one, directly or not, is affected; the test assemblies among them are the ones worth running.
+Changes to scenes, prefabs, settings or packages can't be traced this way, so they ask for a full run.
+"""
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+
+CODE = (".cs", ".asmdef", ".asmref")
+TRACKED = ("Assets/", "Packages/", "ProjectSettings/")
+PLAYER, EDITOR = "Assembly-CSharp", "Assembly-CSharp-Editor"
+
+
+def _guid(f: Path) -> str | None:
+    meta = f.with_name(f.name + ".meta")
+    m = re.search(r"^guid:\s*(\w+)", meta.read_text(encoding="utf-8"), re.M) if meta.exists() else None
+    return m[1] if m else None
+
+
+def _json(f: Path) -> dict:
+    try:
+        return json.loads(f.read_text(encoding="utf-8-sig"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def _is_test(doc: dict) -> bool:
+    return ("UNITY_INCLUDE_TESTS" in doc.get("defineConstraints", [])
+            or "TestAssemblies" in doc.get("optionalUnityReferences", [])
+            or "nunit.framework.dll" in doc.get("precompiledReferences", [])
+            or "UnityEngine.TestRunner" in doc.get("references", []))
+
+
+def scan(root: Path) -> tuple[dict[str, dict], dict[str, str]]:
+    """({assembly: {refs, test, auto}}, {folder: assembly}) for the project's own assemblies."""
+    asms: dict[str, dict] = {}
+    folders: dict[str, str] = {}
+    by_guid: dict[str, str] = {}
+    files = [f for top in ("Assets", "Packages") for f in (root / top).rglob("*.asmdef")]
+    for f in files:
+        doc = _json(f)
+        name = doc.get("name") or f.stem
+        asms[name] = {"refs": doc.get("references", []), "test": _is_test(doc), "auto": doc.get("autoReferenced", True)}
+        folders[f.parent.relative_to(root).as_posix()] = name
+        if g := _guid(f):
+            by_guid[g] = name
+    for f in (r for top in ("Assets", "Packages") for r in (root / top).rglob("*.asmref")):
+        ref = _json(f).get("reference", "")
+        target = by_guid.get(ref[5:], "") if ref.startswith("GUID:") else ref
+        if target:
+            folders[f.parent.relative_to(root).as_posix()] = target
+    for a in asms.values():
+        a["refs"] = [by_guid.get(r[5:], r) if r.startswith("GUID:") else r for r in a["refs"]]
+    # the predefined assemblies see every auto-referenced asmdef; the Editor one also sees the player one
+    autos = [n for n, a in asms.items() if a["auto"] and not a["test"]]
+    asms[PLAYER] = {"refs": autos, "test": False, "auto": False}
+    asms[EDITOR] = {"refs": [*autos, PLAYER], "test": _has_tests(root, editor=True), "auto": False}
+    return asms, folders
+
+
+def _has_tests(root: Path, editor: bool) -> bool:
+    """Tests can live in Editor folders without an asmdef; look for NUnit there."""
+    for f in (root / "Assets").rglob("*.cs"):
+        if editor == ("Editor" in f.relative_to(root).parts) and "NUnit.Framework" in f.read_text(encoding="utf-8", errors="ignore"):
+            return True
+    return False
+
+
+def owner(path: str, folders: dict[str, str]) -> str:
+    parts = path.split("/")
+    for i in range(len(parts) - 1, 0, -1):
+        if (d := "/".join(parts[:i])) in folders:
+            return folders[d]
+    return EDITOR if "Editor" in parts[:-1] else PLAYER
+
+
+def changed_files(root: Path, base: str) -> list[str]:
+    run = lambda *a: subprocess.run(["git", *a], cwd=root, capture_output=True, text=True, timeout=30)
+    diff = run("diff", "--name-only", "--relative", base)
+    if diff.returncode:
+        raise ValueError(f"git diff against {base!r} failed: {diff.stderr.strip()}")
+    new = run("ls-files", "--others", "--exclude-standard")
+    return [ln for ln in (diff.stdout + new.stdout).splitlines() if ln]
+
+
+def affected(root: Path, changed: list[str]) -> dict:
+    asms, folders = scan(root)
+    hit, unmapped = set(), []
+    for p in changed:
+        p = p.replace("\\", "/").removesuffix(".meta")
+        if p.endswith(CODE) and p.startswith(("Assets/", "Packages/")):
+            if p.endswith((".asmdef", ".asmref")):
+                folders.setdefault(p.rsplit("/", 1)[0], Path(p).stem)  # a new or deleted asmdef
+            hit.add(owner(p, folders))
+        elif p.startswith(TRACKED):
+            unmapped.append(p)
+    users: dict[str, set] = {}
+    for name, a in asms.items():
+        for r in a["refs"]:
+            users.setdefault(r, set()).add(name)
+    seen, todo = set(hit), list(hit)
+    while todo:
+        for u in users.get(todo.pop(), ()):
+            if u not in seen:
+                seen.add(u)
+                todo.append(u)
+    tests = sorted(n for n in seen if asms.get(n, {}).get("test"))
+    return {"run_all": bool(unmapped), "assemblies": tests, "changed_assemblies": sorted(hit),
+            "unmapped": unmapped, "all_tests": sorted(n for n, a in asms.items() if a["test"])}

playcal/baseline.py

@@ -0,0 +1,172 @@
+"""playcal baseline: keep a perf baseline (frame time, GC allocation, any per-frame number) and fail
+only when a new run is slower by more than noise.
+
+Input is per-frame samples: {"frame_ms": [...], "gc_alloc_bytes": [...]}, or a bare list (taken as
+frame_ms). check compares a percentile (p95 by default) and bootstraps the difference: it fails only
+when even the low end of the 95% interval is past the tolerance. One hitchy run on a busy machine
+then shows up as "inconclusive", not as a red build.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from .registry import Module
+
+ROUNDS = 400
+
+
+class BaselineError(Exception):
+    pass
+
+
+def samples(doc) -> dict[str, list[float]]:
+    if isinstance(doc, list):
+        doc = {"frame_ms": doc}
+    if not isinstance(doc, dict):
+        raise BaselineError("samples must be a list of numbers or {metric: [numbers]}")
+    out = {}
+    for k, v in doc.items():
+        if isinstance(v, list) and v and all(isinstance(x, (int, float)) and not isinstance(x, bool) for x in v):
+            out[k] = [float(x) for x in v]
+    if not out:
+        raise BaselineError("no metric with a list of numbers found")
+    return out
+
+
+def pct(xs: list[float], q: float) -> float:
+    s = sorted(xs)
+    i = (len(s) - 1) * q / 100
+    lo = int(i)
+    return s[lo] + (s[min(lo + 1, len(s) - 1)] - s[lo]) * (i - lo)
+
+
+def stats(xs: list[float]) -> dict:
+    return {"n": len(xs), "mean": round(sum(xs) / len(xs), 4), "p50": round(pct(xs, 50), 4),
+            "p95": round(pct(xs, 95), 4), "p99": round(pct(xs, 99), 4), "max": max(xs)}
+
+
+def compare(base: list[float], new: list[float], q: float = 95, tolerance: float = 0.10,
+            floor: float = 0.0, rounds: int = ROUNDS, seed: int = 0) -> dict:
+    """Relative change of the q-th percentile with a bootstrap 95% interval.
+    verdict: "slower" (interval's low end > tolerance and the absolute change > floor),
+    "faster", "same" (interval inside ±tolerance), or "inconclusive"."""
+    b0, n0 = pct(base, q), pct(new, q)
+    rng = random.Random(seed)
+    rel = sorted((pct(rng.choices(new, k=len(new)), q) - pct(rng.choices(base, k=len(base)), q)) / (b0 or 1)
+                 for _ in range(rounds))
+    lo, hi = rel[int(rounds * 0.025)], rel[int(rounds * 0.975) - 1]
+    change = (n0 - b0) / (b0 or 1)
+    if lo > tolerance and n0 - b0 > floor:
+        verdict = "slower"
+    elif hi < -tolerance:
+        verdict = "faster"
+    elif -tolerance <= lo and hi <= tolerance:
+        verdict = "same"
+    else:
+        verdict = "inconclusive"
+    return {"verdict": verdict, f"p{q:g}_base": round(b0, 4), f"p{q:g}_new": round(n0, 4),
+            "change": round(change, 4), "ci95": [round(lo, 4), round(hi, 4)]}
+
+
+def _file(folder: Path, name: str) -> Path:
+    if not re.fullmatch(r"[\w.-]+", name):
+        raise BaselineError(f"baseline names are letters, digits, . _ - only: {name!r}")
+    return folder / f"{name}.json"
+
+
+def _read(src: str):
+    text = sys.stdin.read() if src == "-" else Path(src).read_text(encoding="utf-8")
+    return samples(json.loads(text))
+
+
+def save(folder: Path, name: str, data: dict[str, list[float]], source: str) -> dict:
+    folder.mkdir(parents=True, exist_ok=True)
+    doc = {"source": source, "saved_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
+           "stats": {k: stats(v) for k, v in data.items()}, "samples": data}
+    _file(folder, name).write_text(json.dumps(doc), encoding="utf-8")
+    return {"name": name, "stats": doc["stats"]}
+
+
+def check(folder: Path, name: str, data: dict[str, list[float]], q: float, tolerance: float,
+          floors: dict[str, float], metrics: list[str] | None = None) -> dict:
+    path = _file(folder, name)
+    if not path.exists():
+        raise BaselineError(f"no baseline {name!r} in {folder}; make one with: playcal baseline save {name} FILE")
+    base = json.loads(path.read_text(encoding="utf-8"))["samples"]
+    results, missing = {}, []
+    for m in metrics or sorted(set(base) & set(data)):
+        if m not in base or m not in data:
+            missing.append(m)
+            continue
+        results[m] = compare(base[m], data[m], q, tolerance, floors.get(m, 0.0))
+        results[m]["n"] = [len(base[m]), len(data[m])]
+    if not results:
+        raise BaselineError(f"no metric in both the baseline and the new run (asked for {metrics or 'all'})")
+    verdicts = {r["verdict"] for r in results.values()}
+    return {"ok": "slower" not in verdicts, "verdict": "slower" if "slower" in verdicts else
+            "inconclusive" if "inconclusive" in verdicts else "same_or_faster",
+            "metrics": results, **({"missing": missing} if missing else {})}
+
+
+def _floor(spec: str) -> tuple[str, float]:
+    k, sep, v = spec.partition("=")
+    if not sep:
+        raise argparse.ArgumentTypeError("--floor is METRIC=VALUE, e.g. frame_ms=0.5")
+    return k, float(v)
+
+
+def _add_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("action", choices=["save", "check", "show"],
+                   help="save NAME FILE: keep a run as the baseline. check NAME FILE: compare a new run. show NAME")
+    p.add_argument("name", help="baseline name, e.g. level4-android")
+    p.add_argument("source", nargs="?", default="-", help="samples JSON file, or - for stdin (default)")
+    p.add_argument("--dir", default=".playcal/baselines", help="where baselines live (default .playcal/baselines)")
+    p.add_argument("--percentile", type=float, default=95, help="percentile to compare (default 95)")
+    p.add_argument("--tolerance", type=float, default=10, help="allowed slowdown in percent (default 10)")
+    p.add_argument("--floor", type=_floor, action="append", default=[], metavar="METRIC=VALUE",
+                   help="ignore slowdowns smaller than this in absolute units, e.g. frame_ms=0.5. Repeatable")
+    p.add_argument("--metric", action="append", default=None, help="only these metrics (default: all in both)")
+    p.add_argument("--json", action="store_true", help="print the result as JSON")
+
+
+def _run(args: argparse.Namespace) -> int:
+    folder = Path(args.dir)
+    out: Any
+    try:
+        if args.action == "show":
+            path = _file(folder, args.name)
+            if not path.exists():
+                raise BaselineError(f"no baseline {args.name!r} in {folder}")
+            doc = json.loads(path.read_text(encoding="utf-8"))
+            out = {"name": args.name, "source": doc["source"], "saved_at": doc["saved_at"], "stats": doc["stats"]}
+        elif args.action == "save":
+            out = save(folder, args.name, _read(args.source), args.source)
+        else:
+            out = check(folder, args.name, _read(args.source), args.percentile, args.tolerance / 100,
+                        dict(args.floor), args.metric)
+    except (BaselineError, OSError, json.JSONDecodeError) as e:
+        print(f"playcal baseline: {e}", file=sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps(out, ensure_ascii=False))
+    elif args.action == "check":
+        for m, r in out["metrics"].items():
+            q = f"p{args.percentile:g}"
+            print(f"{m}: {r['verdict']}  {q} {r[q + '_base']} -> {r[q + '_new']} ({r['change']:+.1%}, "
+                  f"95% CI {r['ci95'][0]:+.1%} .. {r['ci95'][1]:+.1%})")
+        print("ok" if out["ok"] else "slower than the baseline")
+    else:
+        for m, s in out["stats"].items():
+            print(f"{m}: n={s['n']} p50={s['p50']} p95={s['p95']} p99={s['p99']} max={s['max']}")
+    return 0 if args.action != "check" or out["ok"] else 1
+
+
+MODULE = Module(name="baseline", summary="keep a perf baseline (p95 frame time, GC, memory) and fail only when a run is worse by more than noise",
+                add_args=_add_args, run=_run)

playcal/calibrate.py

@@ -0,0 +1,171 @@
+"""playcal calibrate: predict the human pass rate of each level from bot runs, calibrated on the
+levels a small human panel played.
+
+Both sides are reduced to a per-level attempt pass rate on the logit scale. Human logits are fit to
+bot logits with a random-effects meta-regression (weighted least squares, DerSimonian-Laird tau^2),
+so the between-level scatter the bots don't explain widens every prediction. Leave-one-out over the
+calibration levels says how far to trust it.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import statistics
+import sys
+from pathlib import Path
+
+from .ingest import check_lines
+from .registry import Module
+from .report import _level_sort_key, summarize
+
+# two-sided 95% t quantiles; above 30 df the normal value is close enough
+T975 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, 2.145, 2.131,
+        2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042]
+
+
+def t975(df: int) -> float:
+    return T975[df - 1] if 1 <= df <= len(T975) else 1.96
+
+
+def logit(k: int, n: int) -> tuple[float, float]:
+    """Empirical logit of k/n with the 0.5 correction, and its sampling variance."""
+    return math.log((k + 0.5) / (n - k + 0.5)), 1 / (k + 0.5) + 1 / (n - k + 0.5)
+
+
+def expit(x: float) -> float:
+    return 1 / (1 + math.exp(-x))
+
+
+def _wls(xs, ys, ws):
+    """Weighted least squares y = a + b x. Returns (a, b, cov) with cov the 2x2 inverse of X'WX."""
+    sw = sum(ws)
+    sx = sum(w * x for w, x in zip(ws, xs))
+    sxx = sum(w * x * x for w, x in zip(ws, xs))
+    sy = sum(w * y for w, y in zip(ws, ys))
+    sxy = sum(w * x * y for w, x, y in zip(ws, xs, ys))
+    det = sw * sxx - sx * sx
+    if det <= 1e-12:
+        raise ValueError("the bot pass rates of the calibration levels are all the same; nothing to fit")
+    b = (sw * sxy - sx * sy) / det
+    a = (sy - b * sx) / sw
+    return a, b, ((sxx / det, -sx / det), (-sx / det, sw / det))
+
+
+def fit(points: list[tuple[float, float, float]]) -> dict:
+    """points: (bot logit x, human logit y, human sampling variance v), one per calibration level."""
+    if len(points) < 3:
+        raise ValueError(f"need humans on at least 3 levels to calibrate, got {len(points)}")
+    xs, ys, vs = zip(*points)
+    w = [1 / v for v in vs]
+    a, b, cov = _wls(xs, ys, w)
+    q = sum(wi * (y - a - b * x) ** 2 for wi, x, y in zip(w, xs, ys))
+    df = len(points) - 2
+    # C = sum(w) - trace((X'WX)^-1 X'W^2X)
+    s2 = (sum(wi ** 2 for wi in w), sum(wi ** 2 * x for wi, x in zip(w, xs)), sum(wi ** 2 * x * x for wi, x in zip(w, xs)))
+    c = sum(w) - (cov[0][0] * s2[0] + 2 * cov[0][1] * s2[1] + cov[1][1] * s2[2])
+    tau2 = max(0.0, (q - df) / c) if c > 0 else 0.0
+    a, b, cov = _wls(xs, ys, [1 / (v + tau2) for v in vs])
+    return {"a": a, "b": b, "cov": cov, "tau2": tau2, "df": df}
+
+
+def predict(model: dict, x: float, extra_var: float = 0.0) -> tuple[float, float, float]:
+    """Predicted human pass rate for a level with bot logit x, and its 95% interval. extra_var adds
+    sampling noise when the interval should cover an observed rate rather than the true one."""
+    (c00, c01), (_, c11) = model["cov"]
+    eta = model["a"] + model["b"] * x
+    se = math.sqrt(c00 + 2 * c01 * x + c11 * x * x + model["tau2"] + extra_var)
+    t = t975(model["df"])
+    return expit(eta), expit(eta - t * se), expit(eta + t * se)
+
+
+def leave_one_out(points) -> dict | None:
+    if len(points) < 4:
+        return None
+    hits, errs = 0, []
+    for i, (x, y, v) in enumerate(points):
+        try:
+            m = fit(points[:i] + points[i + 1:])
+        except ValueError:
+            return None
+        p, lo, hi = predict(m, x, extra_var=v)
+        hits += lo <= expit(y) <= hi
+        errs.append(abs(p - expit(y)))
+    return {"levels": len(points), "inside_interval": hits, "median_abs_error": round(statistics.median(errs), 4)}
+
+
+def _stats(files: list[str]) -> dict:
+    lines = [(f, line) for f in files for line in Path(f).read_text(encoding="utf-8").splitlines()]
+    res = check_lines(lines)
+    if not res.accepted:
+        why = f" ({len(res.rejected)} rejected, first: {res.rejected[0]['reason']})" if res.rejected else ""
+        raise ValueError(f"no valid events in {', '.join(files)}{why}")
+    return {(s.game_version, s.level): s for s in summarize(res.accepted)}
+
+
+def calibrate(human: dict, bot: dict, min_attempts: int = 5) -> dict:
+    points, keys = [], []
+    for key in sorted(bot, key=_level_sort_key):
+        h, b = human.get(key), bot[key]
+        if h and h.attempts >= min_attempts and b.attempts:
+            x, _ = logit(b.attempts_passed, b.attempts)
+            y, v = logit(h.attempts_passed, h.attempts)
+            points.append((x, y, v))
+            keys.append(key)
+    model = fit(points)
+    levels = []
+    for key in sorted(bot, key=_level_sort_key):
+        b = bot[key]
+        if not b.attempts:
+            continue
+        p, lo, hi = predict(model, logit(b.attempts_passed, b.attempts)[0])
+        row = {"game_version": key[0], "level": key[1], "bot_pass_rate": round(b.attempts_passed / b.attempts, 4),
+               "bot_attempts": b.attempts, "predicted": round(p, 4), "ci": [round(lo, 4), round(hi, 4)],
+               "calibration": key in keys}
+        h = human.get(key)
+        if h and h.attempts:
+            row["human_pass_rate"], row["human_attempts"] = round(h.attempts_passed / h.attempts, 4), h.attempts
+        levels.append(row)
+    return {"model": {"a": round(model["a"], 4), "b": round(model["b"], 4), "tau": round(math.sqrt(model["tau2"]), 4),
+                      "calibration_levels": len(points)},
+            "leave_one_out": leave_one_out(points), "levels": levels}
+
+
+def _add_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--human", nargs="+", required=True, help="event JSONL from the human panel (accepted.jsonl from ingest)")
+    p.add_argument("--bot", nargs="+", required=True, help="event JSONL from bot runs, same schema")
+    p.add_argument("--min-attempts", type=int, default=5, help="human attempts a level needs to count for calibration (default 5)")
+    p.add_argument("--json", action="store_true", help="print JSON instead of a table")
+
+
+def _pct(x: float) -> str:
+    return f"{100 * x:.0f}%"
+
+
+def _run(args: argparse.Namespace) -> int:
+    try:
+        out = calibrate(_stats(args.human), _stats(args.bot), args.min_attempts)
+    except (OSError, ValueError) as e:
+        print(json.dumps({"error": str(e)}) if args.json else f"playcal calibrate: {e}", file=None if args.json else sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps(out, ensure_ascii=False))
+        return 0
+    m, loo = out["model"], out["leave_one_out"]
+    print(f"{'version':<10} {'level':<8} {'bot':>5} {'predicted human':>20} {'human seen':>12}")
+    for r in out["levels"]:
+        pred = f"{_pct(r['predicted'])} [{_pct(r['ci'][0])}-{_pct(r['ci'][1])}]"
+        seen = f"{_pct(r['human_pass_rate'])} n={r['human_attempts']}" if "human_pass_rate" in r else "-"
+        print(f"{r['game_version']:<10} {r['level']:<8} {_pct(r['bot_pass_rate']):>5} {pred:>20} {seen:>12}"
+              + ("  *" if r["calibration"] else ""))
+    print(f"* calibration level ({m['calibration_levels']}). Attempt pass rates; brackets = 95% interval for the level's true human rate.")
+    if loo:
+        print(f"leave-one-out: {loo['inside_interval']}/{loo['levels']} human rates inside their interval, "
+              f"median error {100 * loo['median_abs_error']:.0f} points.")
+    else:
+        print("leave-one-out needs at least 4 calibration levels; treat these numbers as a rough guess.")
+    return 0
+
+
+MODULE = Module(name="calibrate", summary="predict each level's human pass rate from bot runs, calibrated on a small human panel, with 95% intervals",
+                add_args=_add_args, run=_run)

playcal/checkpoint.py

@@ -0,0 +1,172 @@
+"""playcal checkpoint: save the project's files before each agent step, and go back to any save.
+
+Checkpoints are ordinary git commits on a private ref (refs/playcal/checkpoints), written through
+a separate index file. Your branch, HEAD, staging area and stash are never touched, and the ref is
+not pushed unless you push it on purpose. .gitignore is respected, so Unity's Library/ and Temp/
+stay out if your .gitignore says so.
+
+restore only rewrites the files that differ (so Unity doesn't reimport the whole project) and
+always saves a checkpoint of the current state first, so a restore can itself be undone.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from .registry import Module
+
+REF = "refs/playcal/checkpoints"
+IDENTITY = {"GIT_AUTHOR_NAME": "playcal", "GIT_AUTHOR_EMAIL": "playcal@localhost",
+            "GIT_COMMITTER_NAME": "playcal", "GIT_COMMITTER_EMAIL": "playcal@localhost"}
+
+
+class GitError(RuntimeError):
+    pass
+
+
+class Repo:
+    def __init__(self, path: str | os.PathLike = "."):
+        self.root = Path(self._git(["rev-parse", "--show-toplevel"], cwd=path).strip())
+        self.index = Path(self._git(["rev-parse", "--absolute-git-dir"]).strip()) / "playcal-index"
+
+    def _git(self, args, cwd=None, index=False, check=True) -> str:
+        env = {**os.environ, **IDENTITY}
+        if index:
+            env["GIT_INDEX_FILE"] = str(self.index)
+        try:
+            p = subprocess.run(["git", *args], cwd=cwd or self.root, env=env, capture_output=True, text=True,
+                               encoding="utf-8", errors="replace")
+        except OSError as e:  # no git on PATH, or the folder doesn't exist
+            raise GitError(str(e)) from e
+        if check and p.returncode != 0:
+            raise GitError(p.stderr.strip() or f"git {args[0]} failed")
+        return p.stdout
+
+    def rev(self, name: str) -> str | None:
+        out = self._git(["rev-parse", "--verify", "--quiet", f"{name}^{{commit}}"], check=False).strip()
+        return out or None
+
+    def snapshot_tree(self) -> str:
+        """The working tree as a git tree object, built in playcal's own index."""
+        self.index.unlink(missing_ok=True)
+        if self.rev("HEAD"):
+            self._git(["read-tree", "HEAD"], index=True)  # start from HEAD so add -A only hashes what changed
+        self._git(["add", "-A", "--", "."], index=True)
+        return self._git(["write-tree"], index=True).strip()
+
+    def resolve(self, ident: str) -> str:
+        if ident in ("latest", "last"):
+            ident = REF
+        elif ident.startswith("~") and ident[1:].isdigit():
+            ident = f"{REF}~{ident[1:]}"
+        sha = self.rev(ident)
+        if not sha or not self._is_checkpoint(sha):
+            raise GitError(f"no checkpoint {ident!r} (see: playcal checkpoint list)")
+        return sha
+
+    def _is_checkpoint(self, sha: str) -> bool:
+        tip = self.rev(REF)
+        return bool(tip) and subprocess.run(["git", "merge-base", "--is-ancestor", sha, str(tip)], cwd=self.root,
+                                            capture_output=True).returncode == 0
+
+
+def save(repo: Repo, message: str = "") -> dict:
+    tree = repo.snapshot_tree()
+    tip = repo.rev(REF)
+    if tip and repo._git(["rev-parse", f"{tip}^{{tree}}"]).strip() == tree:
+        return {"id": tip[:10], "saved": False, "reason": "unchanged"}
+    head = repo.rev("HEAD")
+    msg = (message or "checkpoint") + (f"\n\nbase: {head}" if head else "")
+    sha = repo._git(["commit-tree", tree, *(["-p", tip] if tip else []), "-m", msg]).strip()
+    repo._git(["update-ref", REF, sha, tip or ""])  # fails if someone else moved the ref meanwhile
+    return {"id": sha[:10], "saved": True, "message": message or "checkpoint"}
+
+
+def list_(repo: Repo, limit: int = 20) -> list[dict]:
+    if not repo.rev(REF):
+        return []
+    out = repo._git(["log", f"-{limit}", "--format=%H%x1f%cI%x1f%s", REF])
+    rows = []
+    for i, line in enumerate(out.splitlines()):
+        sha, when, subject = line.split("\x1f")
+        rows.append({"id": sha[:10], "back": i, "time": when, "message": subject})
+    return rows
+
+
+def changes(repo: Repo, sha: str, tree: str | None = None) -> list[dict]:
+    """What changed from checkpoint `sha` to the working tree (or to `tree`)."""
+    tree = tree or repo.snapshot_tree()
+    out = repo._git(["diff-tree", "-r", "--no-renames", "--name-status", "-z", f"{sha}^{{tree}}", tree])
+    parts = [p for p in out.split("\0") if p]
+    names = {"A": "added", "D": "deleted", "M": "modified", "T": "modified"}
+    return [{"status": names.get(parts[i], parts[i]), "path": parts[i + 1]} for i in range(0, len(parts) - 1, 2)]
+
+
+def restore(repo: Repo, ident: str) -> dict:
+    sha = repo.resolve(ident)
+    before = save(repo, f"before restore to {sha[:10]}")
+    diff = changes(repo, sha, repo._git(["rev-parse", f"{REF}^{{tree}}"]).strip())
+    gone = [d["path"] for d in diff if d["status"] == "added"]
+    back = [d["path"] for d in diff if d["status"] != "added"]
+    for path in gone:
+        (repo.root / path).unlink(missing_ok=True)
+    repo.index.unlink(missing_ok=True)
+    repo._git(["read-tree", sha], index=True)
+    for i in range(0, len(back), 200):  # keep the command line short on Windows
+        repo._git(["checkout-index", "-f", "--", *back[i:i + 200]], index=True)
+    return {"restored": sha[:10], "undo": before["id"], "rewrote": len(back), "removed": len(gone),
+            "files": diff[:50]}
+
+
+def _add_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("action", choices=["save", "list", "diff", "restore"],
+                   help="save the files now, list saves, diff a save against now, restore a save")
+    p.add_argument("target", nargs="?", help="save: a message. diff/restore: checkpoint id, latest, or ~N (N saves back)")
+    p.add_argument("--repo", default=".", help="folder inside the git repo (default: current folder)")
+    p.add_argument("--limit", type=int, default=20, help="how many saves to list (default 20)")
+    p.add_argument("--json", action="store_true", help="print the result as JSON")
+
+
+def _run(args: argparse.Namespace) -> int:
+    out: Any
+    try:
+        repo = Repo(args.repo)
+        if args.action == "save":
+            out = save(repo, args.target or "")
+        elif args.action == "list":
+            out = list_(repo, args.limit)
+        elif args.action == "diff":
+            sha = repo.resolve(args.target or "latest")
+            out = {"id": sha[:10], "changes": changes(repo, sha)}
+        else:
+            if not args.target:
+                print("playcal checkpoint restore: give a checkpoint id, latest or ~N", file=sys.stderr)
+                return 2
+            out = restore(repo, args.target)
+    except GitError as e:
+        print(json.dumps({"error": str(e)}) if args.json else f"playcal checkpoint: {e}", file=sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps(out, ensure_ascii=False))
+    elif args.action == "list":
+        for r in out:
+            print(f"{r['id']}  ~{r['back']:<3} {r['time']}  {r['message']}")
+    elif args.action == "diff":
+        for c in out["changes"]:
+            print(f"{c['status']:<9} {c['path']}")
+        print(f"{len(out['changes'])} files differ from {out['id']}")
+    elif args.action == "save":
+        print(f"{'saved' if out['saved'] else 'nothing changed since'} {out['id']}")
+    else:
+        print(f"restored {out['restored']}: {out['rewrote']} rewritten, {out['removed']} removed. "
+              f"Undo with: playcal checkpoint restore {out['undo']}")
+    return 0
+
+
+MODULE = Module(name="checkpoint", summary="save project files to a private git ref before each step, and restore any save",
+                add_args=_add_args, run=_run)