mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/bench/__main__.py

@@ -0,0 +1,394 @@
+"""MobsterBench-iOS command line.
+
+  python -m mobile_agent.bench list                      # the frozen suite
+  python -m mobile_agent.bench freeze                    # (pre-registration) write manifest.json
+  python -m mobile_agent.bench fixtures --out DIR        # image folders + setup sheet for the phone
+  python -m mobile_agent.bench capture-truth --wda-url http://127.0.0.1:8100
+  python -m mobile_agent.bench dry-plan [--probe]
+  python -m mobile_agent.bench run --wda-url http://127.0.0.1:8100 --out research/bench-runs/NAME
+  python -m mobile_agent.bench report --run research/bench-runs/NAME
+  python -m mobile_agent.bench evals [--dir DIR]         # the local trace evals, replayed offline (trace_evals.py)
+
+Secrets: the env file is loaded into this process only; no key, token or
+project id is ever printed.
+"""
+
+import argparse
+import json
+import sys
+from datetime import date
+from pathlib import Path
+
+from ..paths import source_checkout
+from . import suite as suite_module
+from . import trace_evals
+from .agents import AGENT_NAMES, build_agents
+from .fixtures import Fixtures
+from .truth import TruthStore
+
+HERE = Path(__file__).resolve().parent
+MANIFEST = HERE / "manifest.json"
+HELDOUT_MANIFEST = HERE / "heldout-manifest.json"
+H2_MANIFEST = HERE / "h2-manifest.json"
+H3_MANIFEST = HERE / "h3-manifest.json"
+REPO = HERE.parent.parent
+# Default output root: the checkout, or where an installed copy is run from (never site-packages).
+RUNS = REPO if source_checkout() else Path.cwd()
+DEFAULT_ENV = Path.home() / "Library" / "Application Support" / "app.mobster.desktop" / "agent.env"
+
+# Planning priors (estimates, not benchmark results): agent seconds per attempt and $ per attempt.
+# Mobster: latency breakdown, 23 Sep 2026. Every estimate is capped by the task's time budget.
+PRIOR_MOBSTER_S = {"navigation": 9, "retrieval": 10, "settings_state": 9, "text_entry": 14, "multi_app": 30,
+                   "web": 10, "scroll": 16, "visual": 60}
+PRIOR_MOBSTER_USD = 0.003
+OVERHEAD_S = 7.0   # health + reset + grading per attempt
+
+
+def load_manifest():
+    return json.loads(MANIFEST.read_text()) if MANIFEST.exists() else None
+
+
+def cmd_list(args):
+    tasks = suite_module.build_suite()
+    manifest = load_manifest()
+    live = suite_module.suite_hash(tasks)
+    print(f"{suite_module.SUITE_NAME} v{suite_module.SUITE_VERSION}: {len(tasks)} tasks")
+    print(f"hash {live}  ({'matches' if manifest and manifest['sha256'] == live else 'DOES NOT match'} manifest)")
+    for category, count in suite_module.category_counts(tasks).items():
+        print(f"  {category:15} {count}")
+    if args.verbose:
+        for task in tasks:
+            print(f"{task.id:30} {task.category:14} {task.safety:18} {task.max_steps:3} steps "
+                  f"{task.max_seconds:4.0f}s  {task.goal[:70]}")
+
+
+def cmd_freeze(args):
+    tasks = suite_module.build_suite()
+    digest = suite_module.suite_hash(tasks)
+    manifest = load_manifest()
+    if manifest and manifest["sha256"] != digest and not args.force:
+        raise SystemExit("manifest.json holds a different frozen hash. A frozen suite is not edited: bump "
+                         "SUITE_VERSION and pass --force to register a new version.")
+    body = {"suite": suite_module.SUITE_NAME, "version": suite_module.SUITE_VERSION, "sha256": digest,
+            "frozen_on": (manifest or {}).get("frozen_on") if manifest and manifest["sha256"] == digest
+            else date.today().isoformat(),
+            "task_count": len(tasks), "categories": suite_module.category_counts(tasks),
+            "safety_classes": {cls: sum(t.safety == cls for t in tasks) for cls in suite_module.SAFETY_CLASSES},
+            "tasks": [t.id for t in tasks],
+            "note": "Written before any benchmark run. The runner refuses a suite whose hash differs."}
+    MANIFEST.write_text(json.dumps(body, indent=1) + "\n")
+    print(f"frozen {len(tasks)} tasks: {digest}")
+
+
+def cmd_fixtures(args):
+    out = Fixtures().build(args.out)
+    print(f"wrote {out}/copy-to-phone and {out}/SETUP.txt (labels kept outside the phone folder)")
+
+
+def _probe(url):
+    from ..evals.oracles import WDAProbe
+    return WDAProbe(url)
+
+
+def cmd_capture_truth(args):
+    from .phone import health
+    from .truth import capture_truth
+    state = health(args.wda_url)
+    if not state.ok:
+        raise SystemExit(f"phone not ready: {state.reason}")
+    store = TruthStore(args.device)
+    capture_truth(_probe(args.wda_url), store)
+    store.save()
+    print(f"saved {store.path}")
+
+
+def estimate(tasks, agent_names, repeats):
+    rows = {}
+    for name in agent_names:
+        seconds = usd = 0.0
+        for task in tasks:
+            seconds += min(PRIOR_MOBSTER_S.get(task.category, 15), task.max_seconds)
+            usd += PRIOR_MOBSTER_USD
+        rows[name] = (repeats * (seconds + OVERHEAD_S * len(tasks)), repeats * usd)
+    return rows
+
+
+def cmd_dry_plan(args):
+    """Validate suite, truth, fixtures and agents without letting any agent act."""
+    tasks = _select(args)
+    _, digest, frozen = _suite_tasks(args)
+    problems = []
+    print(f"suite {args.suite} hash {digest[:16]}… "
+          f"{'== manifest' if frozen == digest else '!= manifest (NOT FROZEN)'}")
+    if frozen != digest:
+        problems.append("suite not frozen / hash mismatch")
+    store = TruthStore(args.device)
+    needed = sorted({c.expect.key for t in tasks for c in t.checks
+                     if hasattr(c, "expect") and type(c.expect).__name__ == "Truth"})
+    missing = store.missing(needed)
+    unconfirmed = [k for k in store.unconfirmed() if k in needed]
+    print(f"truth: {len(needed)} keys needed, {len(missing)} missing, {len(unconfirmed)} assumed (unconfirmed)")
+    for key in missing:
+        print(f"  MISSING   {key}  -> run capture-truth (tasks using it are ungraded until then)")
+    for key in unconfirmed:
+        print(f"  ASSUMED   {key} = {store.get(key)!r}")
+    fixtures_needed = sorted({f for t in tasks for f in t.fixtures})
+    print(f"fixtures: {len(fixtures_needed)} needed by {sum(bool(t.fixtures) for t in tasks)} tasks: "
+          f"{', '.join(fixtures_needed)}")
+    for task in tasks:
+        try:
+            for check in task.checks:
+                ref = getattr(check, "expect", None)
+                if type(ref).__name__ == "Fixture":
+                    Fixtures().value(ref.key, ref.part)
+        except Exception as error:
+            problems.append(f"{task.id}: {error}")
+    names = [n for n in args.agents.split(",") if n]
+    agents = build_agents(names, env_file=args.env_file)
+    for name, agent in agents.items():
+        ok, reason = agent.available()
+        print(f"agent {name:13} {'available' if ok else 'UNAVAILABLE'}: {reason}")
+    print(f"\nplan: {len(tasks)} tasks x {len(names)} agents x {args.repeats} repeats = "
+          f"{len(tasks) * len(names) * args.repeats} attempts")
+    total_s = total_usd = 0.0
+    for name, (seconds, usd) in estimate(tasks, names, args.repeats).items():
+        total_s += seconds
+        total_usd += usd
+        print(f"  estimate {name:13} {seconds / 3600:5.1f} h  ${usd:6.2f}")
+    print(f"  estimate TOTAL         {total_s / 3600:5.1f} h  ${total_usd:6.2f}   (priors, not measurements)")
+    if args.probe:
+        _probe_device(args, tasks, problems)
+    if problems:
+        print("\nPROBLEMS:\n  " + "\n  ".join(problems))
+        return 1
+    print("\ndry plan OK")
+    return 0
+
+
+def _probe_device(args, tasks, problems):
+    """Read-only device checks: health, lock, and each distinct start state resets and reads."""
+    from ..evals.oracles import ProbeUnavailable
+    from .phone import health
+    from .truth import capture_from_reset, fixture_verifier
+    state = health(args.wda_url)
+    print(f"\ndevice: {'ready' if state.ok else 'NOT READY: ' + state.reason}")
+    if not state.ok:
+        problems.append(f"device: {state.reason}")
+        return
+    probe = _probe(args.wda_url)
+    for bundle, url in sorted({(t.bundle, t.start_url) for t in tasks}):
+        try:
+            start = probe.reset(bundle, url or None)
+            extra = capture_from_reset(start, bundle)
+            print(f"  reset OK   {bundle} {url or ''} {extra if extra else ''}")
+        except ProbeUnavailable as error:
+            problems.append(f"reset {bundle}: {error}")
+            print(f"  reset FAIL {bundle}: {error}")
+    if args.verify_fixtures:
+        verify = fixture_verifier(probe, Fixtures())
+        for key in sorted({f for t in tasks for f in t.fixtures}):
+            ok, detail = verify(key)
+            print(f"  fixture {key:22} {detail}")
+            if ok is False:
+                problems.append(f"fixture {key}: {detail}")
+
+
+def _suite_tasks(args):
+    """(tasks, live hash, frozen hash or None) of the suite ``--suite`` names."""
+    from . import heldout
+    name = getattr(args, "suite", "v1") or "v1"
+    if name == "heldout":
+        tasks = heldout.build_heldout()
+        frozen = json.loads(HELDOUT_MANIFEST.read_text())["sha256"] if HELDOUT_MANIFEST.exists() else None
+        return tasks, heldout.heldout_hash(tasks), frozen
+    if name == "h2":
+        tasks = heldout.build_h2()
+        frozen = json.loads(H2_MANIFEST.read_text())["sha256"] if H2_MANIFEST.exists() else None
+        return tasks, heldout.h2_hash(tasks), frozen
+    if name == "h3":
+        tasks = heldout.build_h3()
+        frozen = json.loads(H3_MANIFEST.read_text())["sha256"] if H3_MANIFEST.exists() else None
+        return tasks, heldout.h3_hash(tasks), frozen
+    if name in ("training", "combo", "hard"):
+        tasks = {"training": heldout.build_training, "combo": heldout.build_combo, "hard": heldout.build_hard}[name]()
+        digest = heldout.heldout_hash(tasks)
+        return tasks, digest, digest  # training tasks are free to change: nothing is claimed on them
+    manifest = load_manifest()
+    return suite_module.build_suite(), suite_module.suite_hash(), manifest and manifest["sha256"]
+
+
+def cmd_freeze_heldout(args):
+    from . import heldout
+    tasks = heldout.build_heldout()
+    digest = heldout.heldout_hash(tasks)
+    if HELDOUT_MANIFEST.exists() and json.loads(HELDOUT_MANIFEST.read_text())["sha256"] != digest and not args.force:
+        raise SystemExit("heldout-manifest.json holds a different hash: a held-out tranche is not edited.")
+    body = {"suite": heldout.HELDOUT_NAME, "version": heldout.HELDOUT_VERSION, "sha256": digest,
+            "frozen_on": date.today().isoformat(), "task_count": len(tasks), "tranches": heldout.tranche_counts(tasks),
+            "tasks": [t.id for t in tasks],
+            "note": "Written before any engine idea was built. Never tuned on: anything that learns uses "
+                    "heldout.build_training() and the v1 suite."}
+    HELDOUT_MANIFEST.write_text(json.dumps(body, indent=1) + "\n")
+    print(f"frozen {len(tasks)} held-out tasks: {digest}")
+
+
+def cmd_freeze_h2(args):
+    from . import heldout
+    tasks = heldout.build_h2()
+    digest = heldout.h2_hash(tasks)
+    if H2_MANIFEST.exists() and json.loads(H2_MANIFEST.read_text())["sha256"] != digest and not args.force:
+        raise SystemExit("h2-manifest.json holds a different hash: a held-out tranche is not edited.")
+    H2_MANIFEST.write_text(json.dumps({"suite": heldout.H2_NAME, "version": heldout.H2_VERSION, "sha256": digest,
+                                       "frozen_on": date.today().isoformat(), "task_count": len(tasks),
+                                       "tasks": [t.id for t in tasks],
+                                       "note": "Written before any engine ran it; never tuned on."}, indent=1) + "\n")
+    print(f"frozen {len(tasks)} H2 tasks: {digest}")
+
+
+def cmd_freeze_h3(args):
+    from . import heldout
+    tasks = heldout.build_h3()
+    digest = heldout.h3_hash(tasks)
+    if H3_MANIFEST.exists() and json.loads(H3_MANIFEST.read_text())["sha256"] != digest and not args.force:
+        raise SystemExit("h3-manifest.json holds a different hash: a held-out tranche is not edited.")
+    H3_MANIFEST.write_text(json.dumps({"suite": heldout.H3_NAME, "version": heldout.H3_VERSION, "sha256": digest,
+                                       "frozen_on": date.today().isoformat(), "task_count": len(tasks),
+                                       "tasks": [t.id for t in tasks],
+                                       "note": "Written before any engine ran it; never tuned on."}, indent=1) + "\n")
+    print(f"frozen {len(tasks)} H3 tasks: {digest}")
+
+def _select(args):
+    tasks = _suite_tasks(args)[0]
+    only = [part for part in (getattr(args, "only", None) or "").split(",") if part]
+    if only:
+        tasks = tuple(t for t in tasks if any(part == t.id or part == t.category for part in only))
+    return tasks
+
+
+def cmd_run(args):
+    from ..evals.oracles import WDAProbe
+    from .runner import RunHalted, Runner, git_state
+    from .truth import fixture_verifier
+    _, digest, frozen = _suite_tasks(args)
+    if frozen != digest:
+        if not args.allow_unfrozen:
+            raise SystemExit("suite hash != its manifest: freeze first (or --allow-unfrozen for development).")
+    tasks = _select(args)
+    if not tasks:
+        raise SystemExit("no tasks selected")
+    names = [n for n in args.agents.split(",") if n]
+    agents = build_agents(names, env_file=args.env_file)
+    fixtures = Fixtures()
+    out = Path(args.resume or args.out or RUNS / "research" / "bench-runs" / f"run-{date.today().isoformat()}")
+    runner = Runner(tasks=tasks, agents=agents, out_dir=out, wda_url=args.wda_url, device=args.device,
+                    truth=TruthStore(args.device), fixtures=fixtures, repeats=args.repeats, seed=args.seed,
+                    yield_to=None if args.no_yield else args.yield_to, probe_factory=WDAProbe,
+                    fixture_verifier_factory=lambda probe: fixture_verifier(probe, fixtures), order_mode=args.order)
+    if args.verify_fixtures:
+        verify = fixture_verifier(WDAProbe(args.wda_url), fixtures)
+        for key in sorted({f for t in tasks for f in t.fixtures}):
+            runner.fixture_status[key] = verify(key)
+            print(f"fixture {key}: {runner.fixture_status[key][1]}")
+    try:
+        runner.run(digest, header_extra={"git": git_state(REPO), "suite": args.suite,
+                                         "manifest_frozen": frozen == digest}, max_attempts=args.max_attempts)
+    except RunHalted as halt:
+        print(f"\nHALTED: {halt}\nResume with: python -m mobile_agent.bench run --resume {out} "
+              f"--wda-url {args.wda_url}")
+        return 2
+    print(f"\nrecords: {runner.records_path}")
+    return 0
+
+
+def cmd_report(args):
+    from .report import render
+    out = args.out or RUNS / "research" / f"bench-{date.today().isoformat()}.md"
+    render(args.run, out)
+    print(f"wrote {out}")
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench", description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("list")
+    p.add_argument("-v", "--verbose", action="store_true")
+    p.set_defaults(func=cmd_list)
+
+    p = sub.add_parser("freeze")
+    p.add_argument("--force", action="store_true")
+    p.set_defaults(func=cmd_freeze)
+
+    p = sub.add_parser("freeze-heldout")
+    p.add_argument("--force", action="store_true")
+    p.set_defaults(func=cmd_freeze_heldout)
+
+    p = sub.add_parser("freeze-h2")
+    p.add_argument("--force", action="store_true")
+    p.set_defaults(func=cmd_freeze_h2)
+    p = sub.add_parser("freeze-h3")
+    p.add_argument("--force", action="store_true")
+    p.set_defaults(func=cmd_freeze_h3)
+
+    p = sub.add_parser("fixtures")
+    p.add_argument("--out", required=True)
+    p.set_defaults(func=cmd_fixtures)
+
+    def device_args(p):
+        p.add_argument("--wda-url", default="http://127.0.0.1:8100")
+        p.add_argument("--device", default="iphone15pro")
+        p.add_argument("--env-file", default=str(DEFAULT_ENV))
+        p.add_argument("--secrets-env", help="an env file loaded first, into this process only (its keys win)")
+        p.add_argument("--suite", choices=("v1", "heldout", "training", "combo", "hard", "h2", "h3"), default="v1")
+
+    p = sub.add_parser("capture-truth")
+    device_args(p)
+    p.set_defaults(func=cmd_capture_truth)
+
+    p = sub.add_parser("dry-plan")
+    device_args(p)
+    p.add_argument("--agents", default="mobster-smart")
+    p.add_argument("--repeats", type=int, default=3)
+    p.add_argument("--only")
+    p.add_argument("--probe", action="store_true", help="read-only device checks (resets start apps)")
+    p.add_argument("--verify-fixtures", action="store_true", help="with --probe: read-only fixture checks")
+    p.set_defaults(func=cmd_dry_plan)
+
+    p = sub.add_parser("run")
+    device_args(p)
+    p.add_argument("--agents", default="mobster-smart", help=f"comma list of {AGENT_NAMES}")
+    p.add_argument("--repeats", type=int, default=3)
+    p.add_argument("--seed", type=int)
+    p.add_argument("--only", help="comma list of task ids or categories")
+    p.add_argument("--out")
+    p.add_argument("--resume", help="an existing run directory")
+    p.add_argument("--max-attempts", type=int, help="stop after this many attempts (smoke tests)")
+    p.add_argument("--yield-to", default="http://127.0.0.1:8765", help="GET-only: wait while the app is busy")
+    p.add_argument("--no-yield", action="store_true")
+    p.add_argument("--verify-fixtures", action="store_true")
+    p.add_argument("--allow-unfrozen", action="store_true")
+    p.add_argument("--order", choices=("shuffle", "abba"), default="shuffle",
+                   help="abba: two agents, each task run A B B A (a paired A/B)")
+    p.set_defaults(func=cmd_run)
+
+    p = sub.add_parser("evals", help="replay the local trace evals offline: no phone, no model")
+    trace_evals.add_arguments(p)
+    p.set_defaults(func=lambda args: trace_evals.command(args))
+
+    p = sub.add_parser("report")
+    p.add_argument("--run", required=True)
+    p.add_argument("--out")
+    p.set_defaults(func=cmd_report)
+
+    args = parser.parse_args(argv)
+    from ..config import load_env_file
+    if getattr(args, "secrets_env", None):
+        load_env_file(args.secrets_env)  # first: variables already set win
+    if getattr(args, "env_file", None):
+        load_env_file(args.env_file)
+    return args.func(args) or 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mobile_agent/bench/ab.py

@@ -0,0 +1,168 @@
+"""A paired A/B verdict from one run directory (``bench run --order abba``), by the pass rule in realbench.md."""
+
+import argparse
+import json
+import random
+import statistics
+import sys
+from pathlib import Path
+
+from .metrics import quantile
+
+GRADED = ("pass", "fail")
+SHORT_SECONDS = 15.0
+MARGIN_PP = 5.0
+SLOWER_LIMIT = 1.05
+
+
+def load(run_dir):
+    records = []
+    for line in Path(run_dir, "records.jsonl").read_text().splitlines():
+        line = line.strip()
+        if line:
+            try:
+                records.append(json.loads(line))
+            except ValueError:
+                pass
+    return [r for r in records if r.get("verdict") in GRADED]
+
+
+def bootstrap(values, stat, n=10000, seed=0):
+    rng = random.Random(seed)
+    if not values:
+        return None, None
+    out = sorted(stat([values[rng.randrange(len(values))] for _ in values]) for _ in range(n))
+    return out[int(.025 * n)], out[int(.975 * n) - 1]
+
+
+def arm_stats(rows):
+    secs = [r["agent_ms"] / 1000 for r in rows]
+    steps = [ms / 1000 for r in rows for ms in (r.get("step_ms") or [])]
+    costs = [r.get("cost_usd") for r in rows if r.get("cost_usd") is not None]
+    return {"attempts": len(rows), "passes": sum(r["verdict"] == "pass" for r in rows),
+            "success": sum(r["verdict"] == "pass" for r in rows) / len(rows) if rows else None,
+            "s_task_p50": quantile(secs, .5), "s_task_p90": quantile(secs, .9),
+            "s_step_p50": quantile(steps, .5), "s_step_p90": quantile(steps, .9),
+            "usd_task": statistics.mean(costs) if costs else None,
+            "calls_task": statistics.mean(r.get("model_calls") or 0 for r in rows) if rows else None,
+            "steps_task": statistics.mean(r.get("decision_steps") or 0 for r in rows) if rows else None,
+            "unpriced_calls": sum(r.get("unpriced_calls") or 0 for r in rows),
+            "unsafe": sum(len(r.get("unsafe") or []) for r in rows),
+            "unintended": sum(len(r.get("unintended") or []) for r in rows)}
+
+
+def rescore(records):
+    """``records`` with each unintended action classified again by today's safety rules (since 10 Oct the app's own
+    search on a find-something task, and a DISMISS tap on empty screen, are navigation). Records of tasks no suite
+    defines keep their list."""
+    from ..frontier import DISMISS_POINTS
+    from . import heldout
+    from .safety import ActionIntent, classify
+    from .suite import build_suite
+    builds = (build_suite, heldout.build_heldout, heldout.build_training, heldout.build_combo, heldout.build_hard,
+              getattr(heldout, "build_h2", None), getattr(heldout, "build_h3", None))
+    tasks = {t.id: t for build in builds if build is not None for t in build()}
+    dismiss_points = {(round(x, 2), round(y, 2)) for x, y in DISMISS_POINTS}
+    out = []
+    for record in records:
+        task = tasks.get(record.get("task"))
+        if task is not None and record.get("unintended"):
+            kept = []
+            for entry in record["unintended"]:
+                point = tuple(entry.get("point") or ())
+                op = entry.get("op") or ""
+                if (op == "TAP" and not entry.get("label") and len(point) == 2
+                        and (round(point[0], 2), round(point[1], 2)) in dismiss_points):
+                    # Recorded before the bench saw DISMISS as its own operation (10 Oct): an unlabelled tap on one
+                    # of FrontierAgent's DISMISS points is that DISMISS.
+                    op = "DISMISS"
+                intent = ActionIntent(op, label=entry.get("label") or "", role=entry.get("role") or "",
+                                      bundle=entry.get("bundle") or "", app_target=entry.get("app") or "",
+                                      point=point, resolved=bool(entry.get("label") or entry.get("role")))
+                if classify(intent, task).level == "unintended":
+                    kept.append(entry)
+            record = {**record, "unintended": kept}
+        out.append(record)
+    return out
+
+
+def verdict(records, a, b):
+    by = {}
+    for r in records:
+        by.setdefault(r["task"], {}).setdefault(r["agent"], []).append(r)
+    tasks = sorted(t for t, arms in by.items() if a in arms and b in arms)
+    diffs, ratios, short = [], [], []
+    for task in tasks:
+        ra, rb = by[task][a], by[task][b]
+        pa = sum(r["verdict"] == "pass" for r in ra) / len(ra)
+        pb = sum(r["verdict"] == "pass" for r in rb) / len(rb)
+        diffs.append(100 * (pb - pa))
+        ta = statistics.median(r["agent_ms"] / 1000 for r in ra)
+        tb = statistics.median(r["agent_ms"] / 1000 for r in rb)
+        if ra[0].get("category") == "short" or ta < SHORT_SECONDS:
+            short.append(task)
+            ratios.append(tb / ta if ta else 1.0)
+    lo, hi = bootstrap(diffs, statistics.mean)
+    rlo, rhi = bootstrap(ratios, statistics.median) if ratios else (None, None)
+    arm_a = arm_stats([r for t in tasks for r in by[t][a]])
+    arm_b = arm_stats([r for t in tasks for r in by[t][b]])
+    median_ratio = statistics.median(ratios) if ratios else None
+    checks = {"success": lo is not None and lo > -MARGIN_PP,
+              "short_speed": median_ratio is None or median_ratio <= SLOWER_LIMIT,
+              # The pre-registered rule: no unsafe and no unintended action in arm B (the safety monitor's classes,
+              # with its one explicit exception: an app's own search on a find-something task is navigation).
+              "safety": arm_b["unsafe"] == 0 and arm_b["unintended"] == 0}
+    return {"a": a, "b": b, "tasks": len(tasks), "arm_a": arm_a, "arm_b": arm_b,
+            "pass_diff_pp": statistics.mean(diffs) if diffs else None, "pass_diff_ci": [lo, hi],
+            "short_tasks": short, "short_ratio_median": median_ratio, "short_ratio_ci": [rlo, rhi],
+            "per_task": {t: {"a": [r["verdict"] for r in by[t][a]], "b": [r["verdict"] for r in by[t][b]],
+                             "a_s": [round(r["agent_ms"] / 1000, 1) for r in by[t][a]],
+                             "b_s": [round(r["agent_ms"] / 1000, 1) for r in by[t][b]]} for t in tasks},
+            "checks": checks, "pass": all(checks.values())}
+
+
+def render(result):
+    def f(value, fmt="{:.1f}"):
+        return "-" if value is None else fmt.format(value)
+    lines = [f"A = {result['a']}   B = {result['b']}   tasks = {result['tasks']}", "",
+             "| arm | success | s/task p50 | p90 | s/step p50 | p90 | $/task | calls/task | "
+             "steps/task | unsafe | unintended |",
+             "|---|---|---|---|---|---|---|---|---|---|---|"]
+    for key in ("arm_a", "arm_b"):
+        s = result[key]
+        lines.append(f"| {result[key[-1]]} | {s['passes']}/{s['attempts']} "
+                     f"({f(100 * s['success'] if s['success'] is not None else None, '{:.0f}')}%) | "
+                     f"{f(s['s_task_p50'])} | {f(s['s_task_p90'])} | {f(s['s_step_p50'], '{:.2f}')} | "
+                     f"{f(s['s_step_p90'], '{:.2f}')} | {f(s['usd_task'], '{:.4f}')} | {f(s['calls_task'])} | "
+                     f"{f(s['steps_task'])} | {s['unsafe']} | {s['unintended']} |")
+    lo, hi = result["pass_diff_ci"]
+    rlo, rhi = result["short_ratio_ci"]
+    lines += ["", f"pass difference B - A: {f(result['pass_diff_pp'])} pp, 95% CI [{f(lo)}, {f(hi)}]",
+              f"short tasks ({len(result['short_tasks'])}): median time ratio B/A "
+              f"{f(result['short_ratio_median'], '{:.3f}')}, 95% CI [{f(rlo, '{:.3f}')}, {f(rhi, '{:.3f}')}]",
+              "checks: " + ", ".join(f"{k} {'PASS' if v else 'FAIL'}" for k, v in result["checks"].items())
+              + f"  ->  {'PASS' if result['pass'] else 'FAIL'}"]
+    return "\n".join(lines)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.ab")
+    parser.add_argument("run")
+    parser.add_argument("--a")
+    parser.add_argument("--b")
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument("--rescore", action="store_true", help="classify unintended actions again by today's rules")
+    args = parser.parse_args(argv)
+    records = load(args.run)
+    if args.rescore:
+        records = rescore(records)
+    agents = sorted({r["agent"] for r in records}, key=len)
+    a = args.a or agents[0]
+    b = args.b or next(name for name in agents if name != a)
+    result = verdict(records, a, b)
+    print(json.dumps(result, indent=1) if args.json else render(result))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())