mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/bench/evals/models.py

@@ -0,0 +1,124 @@
+"""Model calls for the L1 runners: one JSON-answering interface over Claude, OpenAI and Jev, timed and priced, under a
+spend cap kept in a ledger file. Keys come from env files and are never printed or written anywhere.
+
+``ask(model, system, parts, schema)`` returns ``Reply(out, usd, seconds, usage, error)``. ``parts``: text strings or
+``("image", path)`` tuples. ``Spend`` refuses a call once the ledger's total plus this run's spend reaches the cap.
+"""
+
+import dataclasses
+import json
+import os
+import pathlib
+import threading
+import time
+
+from ... import frontier, model_clients
+
+
+@dataclasses.dataclass
+class Reply:
+    out: dict = None
+    usd: float = 0.0
+    seconds: float = 0.0
+    usage: dict = None
+    error: str = None
+
+
+class OverBudget(RuntimeError):
+    pass
+
+
+class Spend:
+    """A run's spend, checked against ``cap`` together with every earlier run in ``ledger`` (JSONL of {"usd": x})."""
+
+    def __init__(self, ledger, cap, label):
+        self.ledger, self.cap, self.label = pathlib.Path(ledger).expanduser() if ledger else None, cap, label
+        self.before = self._total()
+        self.usd = 0.0
+        self.lock = threading.Lock()
+
+    def _total(self):
+        if not self.ledger or not self.ledger.exists():
+            return 0.0
+        total = 0.0
+        for line in self.ledger.read_text().splitlines():
+            try:
+                total += float(json.loads(line).get("usd") or 0)
+            except ValueError:
+                pass
+        return total
+
+    def check(self):
+        if self.cap is not None and self.before + self.usd >= self.cap:
+            raise OverBudget(f"spend cap ${self.cap:.2f} reached (${self.before + self.usd:.2f} so far)")
+
+    def add(self, usd):
+        with self.lock:
+            self.usd += usd or 0.0
+
+    def close(self, extra=None):
+        if self.ledger and self.usd:
+            self.ledger.parent.mkdir(parents=True, exist_ok=True)
+            with open(self.ledger, "a") as handle:
+                handle.write(json.dumps({"at": time.strftime("%Y-%m-%dT%H:%M:%S"), "what": self.label,
+                                         "usd": round(self.usd, 5), **(extra or {})}) + "\n")
+
+
+def load_keys(paths):
+    """Load env files into this process (values are never echoed)."""
+    from ...config import load_env_file
+    for path in paths or ():
+        path = os.path.expanduser(path)
+        if os.path.exists(path):
+            load_env_file(path)
+
+
+_clients, _lock = {}, threading.Lock()
+
+
+def client(model, reasoning="low"):
+    key = (model, reasoning)
+    with _lock:
+        if key not in _clients:
+            _clients[key] = model_clients.chat_client(model, reasoning=reasoning)
+        return _clients[key]
+
+
+def _image(path):
+    data = pathlib.Path(path).read_bytes()
+    mime = "image/png" if str(path).lower().endswith(".png") else "image/jpeg"
+    return frontier.image_part(mime, data)
+
+
+def messages(system, parts):
+    content = []
+    for part in parts:
+        if isinstance(part, tuple) and part[0] == "image":
+            if part[1]:
+                content.append(_image(part[1]))
+        elif isinstance(part, dict):
+            content.append(part)
+        else:
+            content.append({"type": "text", "text": str(part)})
+    return [{"role": "system", "content": system}, {"role": "user", "content": content}]
+
+
+def ask(model, system, parts, schema, spend=None, reasoning="low", timeout=120):
+    """One call. The model answers JSON matching ``schema``; ``spend`` is charged its list-price cost."""
+    if spend is not None:
+        spend.check()
+    started = time.monotonic()
+    try:
+        out, usage = client(model, reasoning).complete(messages(system, parts), schema, timeout=timeout)
+    except Exception as exc:  # a failed call is a row, not a crash: the scorer counts it
+        return Reply(error=f"{type(exc).__name__}: {str(exc)[:200]}", seconds=time.monotonic() - started)
+    seconds = time.monotonic() - started
+    usd = model_clients.cost_usd(model, usage) or 0.0
+    if spend is not None:
+        spend.add(usd)
+    return Reply(out=out if isinstance(out, dict) else {"value": out}, usd=usd, seconds=seconds, usage=usage)
+
+
+def jev():
+    from ...decision_provider import JevDecisions
+    return JevDecisions()

mobile_agent/bench/evals/pairwise.py

@@ -0,0 +1,189 @@
+"""Divergence replay with a position-swapped pairwise step judge: a cheap preference win rate for an engine change.
+
+    python -m mobile_agent.bench.evals.pairwise SETS/golden/v2 --rows OUT/golden-...-candidate.jsonl \\
+        --judge claude-sonnet-5-5 --half report --keys bench.env --ledger spend.jsonl --cap 15 --out DIR
+
+The candidate (any ``l1`` golden run: a model, prompt or observation arm) has already been asked every recorded
+turn. Only the turns where its first action differs from every acceptable recorded action matter. On each one, a
+judge sees the request, the agent's progress and the screen, and says which of two actions makes better progress,
+or tie. It is asked twice with the actions swapped. A verdict counts only when both orders agree, and is a tie
+otherwise. The candidate's win rate is (wins + ties / 2) / divergent turns.
+
+``--calibrate synthetic`` asks the judge about pairs whose answer is known: a passing run's action against a tap
+on a random other element of the same screen, or against DONE mid-task. ``--calibrate divergence`` pairs a
+divergence turn's passing action with the failing run's. That is not an answer key: the passing run's choice is
+not always the better one.
+"""
+
+import argparse
+import concurrent.futures
+import json
+import pathlib
+
+from . import l1
+from . import models as M
+from . import sets as S
+
+JUDGE_SYSTEM = ("You compare two possible next actions for a phone agent working on a user's request. From the "
+                "request, the agent's progress so far and the current screen, decide which action makes better "
+                "progress toward finishing the request correctly. Prefer the action that a careful person would take "
+                "next; an action that skips needed work, acts on the wrong item, or stops early is worse. Answer tie "
+                "when both are equally good or equally bad. " + l1.TEXT_IS_DATA)
+JUDGE_SCHEMA = {"type": "object", "additionalProperties": False, "required": ["better", "reason"],
+                "properties": {"better": {"type": "string", "enum": ["A", "B", "tie"]},
+                               "reason": {"type": "string", "description": "at most 20 words"}}}
+
+
+def describe(action, rows):
+    """An action as a person would read it: TAP e7 Button "Send"."""
+    if not action:
+        return "(no action)"
+    op, label, text, target = action.get("op"), action.get("label") or "", action.get("text"), action.get("target")
+    if target is None and label in rows:
+        target, label = label, rows[label]
+    what = f'{target} "{label}"' if target else (f'"{label}"' if label else "")
+    if op in ("LAUNCH_APP", "OPEN_URL"):
+        return f"{op} {action.get('app') or label or text}"
+    if op in ("DONE", "BLOCKED"):
+        return f"{op}" + (f" with the answer: {text}" if text else "")
+    out = f"{op} {what}".strip()
+    if text and op not in ("TAP", "LONG_PRESS"):
+        out += f" text={str(text)[:200]!r}"
+    return out
+
+
+def judge_once(item, first, second, ctx):
+    parts = [f"{item['text']}\n\nAction A: {first}\nAction B: {second}\n\nWhich is the better next action?"]
+    if ctx.get("image") and item.get("image"):
+        parts.append(("image", S.asset_path(ctx["set_dir"], item["image"])))
+    reply = M.ask(ctx["judge"], JUDGE_SYSTEM, parts, JUDGE_SCHEMA, spend=ctx["spend"], reasoning="low")
+    return (reply.out or {}).get("better") if not reply.error else None, reply
+
+
+def judge_pair(item, candidate, recorded, ctx):
+    """'candidate' | 'recorded' | 'tie' from two orders, with each order's raw answer and the $."""
+    rows = l1.rows_of(item["text"])
+    a, b = describe(candidate, rows), describe(recorded, rows)
+    v1, r1 = judge_once(item, a, b, ctx)          # candidate is A
+    v2, r2 = judge_once(item, b, a, ctx)          # candidate is B
+    first = {"A": "candidate", "B": "recorded", "tie": "tie"}.get(v1)
+    second = {"A": "recorded", "B": "candidate", "tie": "tie"}.get(v2)
+    verdict = first if first == second and first is not None else "tie"
+    return {"verdict": verdict, "orders": [v1, v2], "consistent": first == second and first is not None,
+            "candidate": a, "recorded": b, "usd": r1.usd + r2.usd, "seconds": r1.seconds + r2.seconds,
+            "error": r1.error or r2.error}
+
+
+def synthetic_pairs(items, n, seed=20261010):
+    """Pairs whose answer is known: a passing run's recorded action against a tap on a random other element of the
+    same screen, or against DONE in the middle of the task. The judge should prefer the recorded action."""
+    import random
+    rng = random.Random(seed)
+    pool = [i for i in items if i["kind"] == "path"]
+    rng.shuffle(pool)
+    out = []
+    for item in pool:
+        if len(out) >= n:
+            break
+        rows = l1.rows_of(item["text"])
+        if len(out) % 2 == 0:
+            taken = {item["gold"].get("target")} | {a[4] for a in item.get("alternatives") or () if len(a) > 4}
+            taken |= {eid for eid, label in rows.items() if T_norm(label) in {T_norm(item["gold"]["label"])} |
+                      {a[1] for a in item.get("alternatives") or ()}}
+            others = [eid for eid, label in rows.items() if eid not in taken and label]
+            if not others:
+                continue
+            eid = rng.choice(others)
+            bad = {"op": "TAP", "label": rows[eid], "text": None, "app": None, "target": eid}
+        else:
+            bad = {"op": "DONE", "label": "", "text": "Done.", "app": None, "target": None}
+        out.append((item, bad, item["gold"]))
+    return out
+
+
+def T_norm(text):
+    from .traces import norm
+    return norm(text)
+
+
+def replay(set_dir, rows_path, judge, half="report", workers=4, ledger=None, cap=None, image=True, calibrate=None,
+           calibrate_n=20):
+    set_dir = pathlib.Path(set_dir).expanduser()
+    _, items = S.load(set_dir, half=half if half != "all" else None)
+    by = {i["id"]: i for i in items}
+    spend = M.Spend(ledger, cap, f"pairwise {judge} {'calibrate' if calibrate else pathlib.Path(rows_path).name}")
+    ctx = {"set_dir": set_dir, "judge": judge, "spend": spend, "image": image}
+    jobs = []
+    if calibrate == "divergence":
+        for item in items:
+            if item.get("wrong"):
+                jobs.append((item, item["wrong"], item["gold"]))   # the "candidate" is the failing run's action
+    elif calibrate:
+        jobs = synthetic_pairs(items, calibrate_n)
+    else:
+        for line in pathlib.Path(rows_path).expanduser().read_text().splitlines():
+            row = json.loads(line)
+            item = by.get(row["id"])
+            if item is None or row.get("error") or row.get("skipped"):
+                continue
+            if not l1.golden_match(row, item)["alt"]:
+                jobs.append((item, row.get("pred"), item["gold"]))
+    try:
+        with concurrent.futures.ThreadPoolExecutor(workers) as pool:
+            results = list(pool.map(lambda job: {"id": job[0]["id"], "step_type": job[0]["step_type"],
+                                                 "kind": job[0]["kind"], **judge_pair(job[0], job[1], job[2], ctx)},
+                                    jobs))
+    finally:
+        spend.close({"pairs": len(jobs)})
+    n = len(results)
+    wins = sum(r["verdict"] == "candidate" for r in results)
+    losses = sum(r["verdict"] == "recorded" for r in results)
+    ties = n - wins - losses
+    summary = {"rows": str(rows_path) if not calibrate else "calibrate", "judge": judge, "image": image, "half": half,
+               "divergent": n, "candidate_wins": wins, "recorded_wins": losses, "ties": ties,
+               "win_rate": round((wins + ties / 2) / n, 3) if n else None,
+               "consistent": round(sum(r["consistent"] for r in results) / n, 3) if n else None,
+               "usd": round(sum(r["usd"] for r in results), 4), "errors": sum(bool(r["error"]) for r in results)}
+    if not calibrate:
+        scored = sum(1 for line in pathlib.Path(rows_path).expanduser().read_text().splitlines()
+                     if json.loads(line)["id"] in by and not json.loads(line).get("skipped"))
+        summary["turns"] = scored
+        summary["divergence_rate"] = round(n / scored, 3) if scored else None
+    return summary, results
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.evals.pairwise",
+                                     description=__doc__.split("\n\n")[0])
+    parser.add_argument("set_dir")
+    parser.add_argument("--rows", help="a golden run's rows (the candidate)")
+    parser.add_argument("--calibrate", choices=("synthetic", "divergence"),
+                        help="judge known pairs: synthetic (the recorded action against a random other tap or an early "
+                             "DONE) or divergence (a passing run's action against the failing run's)")
+    parser.add_argument("--calibrate-n", type=int, default=20)
+    parser.add_argument("--judge", default="claude-sonnet-5-5")
+    parser.add_argument("--no-image", action="store_true")
+    parser.add_argument("--half", default="report", choices=("tune", "report", "all"))
+    parser.add_argument("--workers", type=int, default=4)
+    parser.add_argument("--keys", nargs="*", default=())
+    parser.add_argument("--ledger")
+    parser.add_argument("--cap", type=float)
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args(argv)
+    if not args.rows and not args.calibrate:
+        parser.error("give --rows or --calibrate")
+    M.load_keys(args.keys)
+    summary, results = replay(args.set_dir, args.rows, args.judge, args.half, args.workers, args.ledger, args.cap,
+                              image=not args.no_image, calibrate=args.calibrate, calibrate_n=args.calibrate_n)
+    out = pathlib.Path(args.out).expanduser()
+    out.mkdir(parents=True, exist_ok=True)
+    name = f"calibrate-{args.calibrate}" if args.calibrate else pathlib.Path(args.rows).stem
+    (out / f"pairwise-{name}-{l1.slug(args.judge)}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in results))
+    with open(out / "pairwise-summary.jsonl", "a") as handle:
+        handle.write(json.dumps(summary) + "\n")
+    print(json.dumps(summary))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mobile_agent/bench/evals/proxy.py

@@ -0,0 +1,90 @@
+"""Proxy validity: does a cheap eval predict the paired A/B? One file of hypotheses, one rank correlation per proxy.
+
+    python -m mobile_agent.bench.evals.proxy validity.jsonl [--outcome pass_delta|rubric_delta] [--min 4]
+
+Each line of the file is one hypothesis that reached (or was stopped before) the paired A/B:
+
+    {"hypothesis": "...", "change": "PR or branch", "date": "2026-10-10",
+     "gates": {"l1": "written before the run", "l2": "..."},
+     "proxies": {"<level>:<set>:<metric>": value, ...},     # the change's delta on each proxy, sign: + is better
+     "l3": {"batch": "ac1", "pass_delta": -0.125, "pass_ci": [-0.25, -0.025], "rubric_delta": -0.033}}
+
+A hypothesis with no "l3" was stopped by a cheaper level, and is listed but not correlated. For every proxy with
+at least ``--min`` hypotheses that have an L3 result, it prints Spearman's rank correlation with the outcome and how
+often the signs agree. A proxy that doesn't predict gets dropped or fixed.
+"""
+
+import argparse
+import json
+import pathlib
+
+
+def ranks(xs):
+    order = sorted(range(len(xs)), key=lambda i: xs[i])
+    out = [0.0] * len(xs)
+    i = 0
+    while i < len(order):
+        j = i
+        while j + 1 < len(order) and xs[order[j + 1]] == xs[order[i]]:
+            j += 1
+        for k in range(i, j + 1):
+            out[order[k]] = (i + j) / 2 + 1
+        i = j + 1
+    return out
+
+
+def spearman(xs, ys):
+    if len(xs) < 3:
+        return None
+    rx, ry = ranks(xs), ranks(ys)
+    mx, my = sum(rx) / len(rx), sum(ry) / len(ry)
+    sxy = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
+    sxx = sum((a - mx) ** 2 for a in rx) ** .5
+    syy = sum((b - my) ** 2 for b in ry) ** .5
+    return round(sxy / (sxx * syy), 3) if sxx and syy else None
+
+
+def validity(entries, outcome="pass_delta", minimum=4):
+    done = [e for e in entries if (e.get("l3") or {}).get(outcome) is not None]
+    keys = sorted({k for e in done for k in (e.get("proxies") or {})})
+    rows = []
+    for key in keys:
+        pairs = [(e["proxies"][key], e["l3"][outcome]) for e in done if (e.get("proxies") or {}).get(key) is not None]
+        row = {"proxy": key, "n": len(pairs)}
+        if len(pairs) >= minimum:
+            xs, ys = zip(*pairs)
+            row["spearman"] = spearman(list(xs), list(ys))
+            row["sign_agreement"] = round(sum((x > 0) == (y > 0) for x, y in pairs if x and y) /
+                                          max(1, sum(1 for x, y in pairs if x and y)), 3)
+        rows.append(row)
+    return {"hypotheses": len(entries), "with_l3": len(done), "stopped_early": len(entries) - len(done),
+            "outcome": outcome, "proxies": rows}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.evals.proxy",
+                                     description=__doc__.split("\n\n")[0])
+    parser.add_argument("file")
+    parser.add_argument("--outcome", default="pass_delta")
+    parser.add_argument("--min", type=int, default=4)
+    args = parser.parse_args(argv)
+    entries = [json.loads(line) for line in pathlib.Path(args.file).expanduser().read_text().splitlines()
+               if line.strip()]
+    report = validity(entries, args.outcome, args.min)
+    print(f"{report['hypotheses']} hypotheses: {report['with_l3']} with a paired A/B, {report['stopped_early']} "
+          f"stopped earlier. Outcome: {args.outcome}.")
+    for e in entries:
+        l3 = e.get("l3") or {}
+        print(f"- {e['hypothesis']}: " + (f"L3 {l3.get('batch')} {args.outcome} {l3.get(args.outcome):+.3f}" if
+                                          l3.get(args.outcome) is not None
+                                          else "no L3 (" + e.get("status", "stopped") + ")"))
+    for row in report["proxies"]:
+        if "spearman" in row:
+            print(f"  {row['proxy']}: n {row['n']}, Spearman {row['spearman']}, signs agree {row['sign_agreement']}")
+        else:
+            print(f"  {row['proxy']}: n {row['n']} (needs {args.min} with a paired A/B)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mobile_agent/bench/evals/sets.py

@@ -0,0 +1,149 @@
+"""Frozen L1 sets: versioned, hashed, split by task into a tune half and a report half.
+
+A set lives in its own folder, outside the repository (its items hold task text and screens):
+
+    <root>/<name>/<version>/items.jsonl    one item per line, sorted by id; each has "id", "task", "half"
+    <root>/<name>/<version>/assets/        the screenshots items name, by content hash
+    <root>/<name>/<version>/MANIFEST.json  counts, sources, the builder's commit, and "sha256"
+
+``sha256`` covers items.jsonl and every asset's bytes. ``load`` refuses a set whose files no longer match it, so a
+number reported on a set is a number on exactly that set. A change tuned on the tune half is reported on the report
+half; the halves split tasks, not items, so no task's screens leak from one half into the other.
+
+The halves come from one task-to-half map shared by every set (``make_halves``): tasks are dealt alternately within
+strata (core or not, category) in the order of a seeded hash. Tasks in a split named in ``forbidden`` (val, test)
+never enter a set.
+"""
+
+import datetime
+import hashlib
+import json
+import pathlib
+import subprocess
+
+SEED = "20261010"
+HALVES = ("tune", "report")
+
+
+def _h(text):
+    return hashlib.sha256(text.encode()).hexdigest()
+
+
+def make_halves(strata, seed=SEED):
+    """{task: "tune" | "report"} from {task: stratum}: within each stratum, tasks in seeded-hash order alternate."""
+    by = {}
+    for task, stratum in strata.items():
+        by.setdefault(str(stratum), []).append(task)
+    out = {}
+    for stratum in sorted(by):
+        for i, task in enumerate(sorted(by[stratum], key=lambda t: _h(f"{seed}:{stratum}:{t}"))):
+            out[task] = HALVES[i % 2]
+    return out
+
+
+def halves_from_splits(splits, extra=None, seed=SEED):
+    """The halves of every dev task in a tuning splits file (core and category strata), plus ``extra``
+    {task: stratum} for tasks outside it (another benchmark's)."""
+    dev = splits["splits"]["dev"]
+    core = set(splits["splits"].get("dev_core") or ())
+    tasks = splits.get("tasks") or {}
+    strata = {t: f"{'core' if t in core else 'dev'}:{(tasks.get(t) or {}).get('category')}" for t in dev}
+    strata.update({t: f"extra:{s}" for t, s in (extra or {}).items()})
+    return make_halves(strata, seed)
+
+
+def forbidden_tasks(splits, names=("val", "test")):
+    return {t for name in names for t in (splits["splits"].get(name) or ())}
+
+
+def _git_head():
+    try:
+        here = pathlib.Path(__file__).resolve().parent
+        return subprocess.run(["git", "-C", str(here), "rev-parse", "--short=12", "HEAD"], capture_output=True,
+                              text=True, timeout=5).stdout.strip() or None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+class SetWriter:
+    """Collects items and assets, then writes a frozen set. ``asset(path)`` copies a file in and returns its name."""
+
+    def __init__(self, root, name, version, halves, forbidden=()):
+        self.dir = pathlib.Path(root).expanduser() / name / version
+        if (self.dir / "MANIFEST.json").exists():
+            raise FileExistsError(f"{self.dir} is frozen; build a new version instead")
+        self.name, self.version, self.halves, self.forbidden = name, version, halves, set(forbidden)
+        self.items, self.assets, self.skipped = [], {}, {"forbidden": 0, "no_half": 0}
+
+    def asset(self, path):
+        if not path:
+            return None
+        path = pathlib.Path(path)
+        try:
+            data = path.read_bytes()
+        except OSError:
+            return None
+        name = hashlib.sha256(data).hexdigest()[:20] + path.suffix.lower()
+        self.assets[name] = data
+        return f"assets/{name}"
+
+    def add(self, item):
+        task = item["task"]
+        if task in self.forbidden:
+            self.skipped["forbidden"] += 1
+            return False
+        half = self.halves.get(task)
+        if half is None:
+            self.skipped["no_half"] += 1
+            return False
+        self.items.append({**item, "half": half})
+        return True
+
+    def write(self, sources=(), notes=""):
+        ids = [i["id"] for i in self.items]
+        if len(ids) != len(set(ids)):
+            raise ValueError("duplicate item ids")
+        self.dir.mkdir(parents=True, exist_ok=True)
+        tmp = self.dir / "assets"
+        tmp.mkdir(exist_ok=True)
+        for name, data in self.assets.items():
+            (tmp / name).write_bytes(data)
+        lines = "".join(json.dumps(i, ensure_ascii=False, sort_keys=True) + "\n" for i in sorted(self.items,
+                                                                                               key=lambda i: i["id"]))
+        (self.dir / "items.jsonl").write_text(lines, encoding="utf-8")
+        manifest = {"name": self.name, "version": self.version,
+                    "created": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
+                    "builder_commit": _git_head(), "items": len(self.items),
+                    "halves": {h: sum(i["half"] == h for i in self.items) for h in HALVES},
+                    "tasks": {h: len({i["task"] for i in self.items if i["half"] == h}) for h in HALVES},
+                    "assets": len(self.assets), "skipped": self.skipped, "sources": list(sources), "notes": notes,
+                    "sha256": digest(self.dir)}
+        (self.dir / "MANIFEST.json").write_text(json.dumps(manifest, indent=1))
+        return manifest
+
+
+def digest(set_dir):
+    """sha256 over items.jsonl and every asset's name and bytes."""
+    set_dir = pathlib.Path(set_dir)
+    h = hashlib.sha256((set_dir / "items.jsonl").read_bytes())
+    assets = set_dir / "assets"
+    for path in sorted(assets.iterdir()) if assets.is_dir() else ():
+        h.update(path.name.encode())
+        h.update(path.read_bytes())
+    return h.hexdigest()
+
+
+def load(set_dir, half=None, verify=True):
+    """(manifest, items) of a frozen set, refusing one whose files no longer match its hash."""
+    set_dir = pathlib.Path(set_dir).expanduser()
+    manifest = json.loads((set_dir / "MANIFEST.json").read_text())
+    if verify and digest(set_dir) != manifest["sha256"]:
+        raise ValueError(f"{set_dir} no longer matches its MANIFEST sha256: rebuild it as a new version")
+    items = [json.loads(line) for line in (set_dir / "items.jsonl").read_text(encoding="utf-8").splitlines() if line]
+    if half:
+        items = [i for i in items if i["half"] == half]
+    return manifest, items
+
+
+def asset_path(set_dir, name):
+    return pathlib.Path(set_dir).expanduser() / name if name else None