mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/bench/guard_replay.py

@@ -0,0 +1,129 @@
+"""Offline check of a change to the commit guard's families: what would it newly allow, on recorded runs?
+
+    python -m mobile_agent.bench.guard_replay --runs DIR... --model claude-sonnet-5-5 --compiles 2 --out OUT.json \\
+        --env-file bench.env
+
+For each recorded task, the request's contract is compiled ``--compiles`` times (contract.compile_contract, as the
+loop does). Every recorded gate decision is then replayed under the old and the new family rule: each control the
+guard refused, each commit-class control the run tapped, and every commit-class control on every screen the model
+read. The guard's other checks (amount bounds, typed text, object names, approvals) are unchanged by a family change
+and are not replayed: the question is only which controls' verbs would now count as declared.
+
+A control newly allowed is listed with whether the request asks for its act (iosworld_stats.commit_requested, an
+approximation) so each can be read by hand. The change passes when no newly allowed control is unrequested.
+"""
+
+import argparse
+import collections
+import json
+import pathlib
+import re
+
+from .. import contract as K
+from .. import contract_screen as KS
+from .iosworld_stats import NON_GUARD_REFUSALS, commit_requested
+
+ROW = re.compile(r'^e\d+ (\w+) "([^"]*)"', re.M)
+OTHER_REFUSALS = NON_GUARD_REFUSALS | {"tap_xy", "untargeted_swipe", "disabled", "passcode_keypad", "needs_approval"}
+
+
+def old_family(item):
+    """The family rule before the request's own word for the act counted (contract.py at e24a7f9f)."""
+    fam = set(KS.FAMILIES.get(item.act, ()))
+    if item.act == "other":
+        fam |= {v.casefold() for v in KS.COMMIT_WORDS.findall(item.quote)}
+    return fam
+
+
+def declared(contract, verbs, app, family):
+    return any(i.kind == "COMMIT" and K.Contract._app_ok(i, app) and verbs <= family(i) for i in contract.items)
+
+
+def forbidden(contract, verbs, app):
+    return any(i.kind == "FORBID" and K.Contract._app_ok(i, app) and verbs & i.family() for i in contract.items)
+
+
+def decisions(folder):
+    """(goal, app names, [(kind, label, role, app)]) from one recorded run."""
+    record = json.loads((folder / "task.json").read_text())
+    prompts = [json.loads(line) for line in (folder / "prompts.jsonl").read_text().splitlines() if line.strip()]
+    app_at, names, out = {}, [], []
+    for p in prompts:
+        text = p.get("text") or ""
+        current = text.split("Current app: ", 1)[1].split("\n", 1)[0].strip() if "Current app: " in text else ""
+        app_at[p.get("step")] = current
+        if not names and "Apps you may use:\n" in text:
+            names = [line.split(": ")[0].lstrip("- ").strip() for line in
+                     text.split("Apps you may use:\n", 1)[1].split("\n\n")[0].splitlines()]
+        at = text.find("Screen elements:")
+        for role, label in ROW.findall(text[at:] if at >= 0 else ""):
+            if KS.label_verbs(label, role):
+                out.append(("on screen", label, role, current))
+    for e in (record.get("mobster") or {}).get("events") or ():
+        if e.get("event") == "frontier_refused" and e.get("label") not in OTHER_REFUSALS:
+            out.append(("refused", e.get("label") or "", "Button", app_at.get(e.get("step"), "")))
+        elif e.get("event") == "frontier_action" and e.get("operation") in ("TAP", "LONG_PRESS") and KS.label_verbs(
+                e.get("target_label") or ""):
+            out.append(("tapped", e.get("target_label") or "", "Button", e.get("app_name")
+                        or app_at.get(e.get("step"), "")))
+    return record.get("goal") or "", names, out
+
+
+def main(argv=None):
+    from ..model_clients import chat_client, cost_usd
+    from .iosworld import load_envs
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.guard_replay")
+    parser.add_argument("--runs", nargs="+", required=True, help="run folders (each holding NNN-task/ folders)")
+    parser.add_argument("--model", default="claude-sonnet-5-5")
+    parser.add_argument("--compiles", type=int, default=2)
+    parser.add_argument("--env-file", action="append")
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args(argv)
+    load_envs(args.env_file)
+    client = chat_client(args.model, reasoning="low")
+    by_task = collections.defaultdict(list)
+    for root in args.runs:
+        for task_json in sorted(pathlib.Path(root).glob("*/task.json")):
+            if (task_json.parent / "prompts.jsonl").exists():
+                by_task[task_json.parent.name.split("-", 1)[1]].append(task_json.parent)
+    newly, counts, compiled = [], collections.Counter(), {}
+    for task, folders in sorted(by_task.items()):
+        goal, names, _ = decisions(folders[0])
+        contracts = []
+        for _ in range(args.compiles):
+            c = K.compile_contract(client, goal, names)
+            if c is not None:
+                contracts.append(c)
+        compiled[task] = [[f"{i.kind} {i.act} ({i.app}): {i.what} | quote={i.quote!r}" for i in c.items if i.kind in
+                           ("COMMIT", "FORBID")] for c in contracts]
+        seen = set()
+        for folder in folders:
+            _, _, decided = decisions(folder)
+            for kind, label, role, app in decided:
+                verbs = frozenset(KS.label_verbs(label, role))
+                if not verbs:
+                    continue
+                for index, c in enumerate(contracts):
+                    counts["decisions"] += 1
+                    old = declared(c, verbs, app, old_family) and not forbidden(c, verbs, app)
+                    new = declared(c, verbs, app, lambda i: i.family()) and not forbidden(c, verbs, app)
+                    counts[f"{kind}: old {'allow' if old else 'refuse'} -> new {'allow' if new else 'refuse'}"] += 1
+                    if new and not old and (task, label, app, index) not in seen:
+                        seen.add((task, label, app, index))
+                        newly.append({"task": task, "kind": kind, "label": label, "app": app, "compile": index,
+                                      "verbs": sorted(verbs), "requested": commit_requested(label, goal),
+                                      "goal": goal})
+    usd = cost_usd(client.model, client.usage) or 0.0
+    result = {"tasks": len(by_task), "counts": dict(counts), "newly_allowed": newly,
+              "newly_allowed_unrequested": [n for n in newly if n["requested"] is not True],
+              "compiled": compiled, "usd": round(usd, 4)}
+    pathlib.Path(args.out).write_text(json.dumps(result, indent=1, ensure_ascii=False))
+    print(json.dumps({k: result[k] for k in ("tasks", "counts", "usd")}, indent=1))
+    for n in newly:
+        print(f"NEWLY ALLOWED {n['task']:14} {n['kind']:9} {n['app']:10} {n['label'][:60]!r} "
+              f"requested={n['requested']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mobile_agent/bench/h2-manifest.json

@@ -0,0 +1,20 @@
+{
+ "suite": "MobsterBench-iOS H2",
+ "version": "1.0",
+ "sha256": "a5d20b227eab62cfdbf39c4a80adbebcf19490311669e3a6f5e0f2edaa2ddf44",
+ "frozen_on": "2026-10-10",
+ "task_count": 10,
+ "tasks": [
+  "h2.calendar_three",
+  "h2.contact_two",
+  "h2.about_two",
+  "h2.receipts_count",
+  "h2.room_plus_phone",
+  "h2.code_digit_sum",
+  "h2.maps_alcatraz",
+  "h2.bridge_opened",
+  "h2.turing_phd_university",
+  "h2.reminder_last_alpha"
+ ],
+ "note": "Written before any engine ran it; never tuned on."
+}

mobile_agent/bench/h3-manifest.json

@@ -0,0 +1,20 @@
+{
+ "suite": "MobsterBench-iOS H3",
+ "version": "1.0",
+ "sha256": "559d23a39a5887523fcfc8ba63d816c2935e71f2ffda8e871d8777a93204a1c3",
+ "frozen_on": "2026-10-10",
+ "task_count": 10,
+ "tasks": [
+  "h3.hopper_retired",
+  "h3.lovelace_father",
+  "h3.eiffel_height",
+  "h3.apollo_cmp",
+  "h3.python_designer",
+  "h3.coit_city",
+  "h3.space_needle_two",
+  "h3.sunglasses_count",
+  "h3.locker_and_room",
+  "h3.phone_minus_room"
+ ],
+ "note": "Written before any engine ran it; never tuned on."
+}

mobile_agent/bench/heldout-manifest.json

@@ -0,0 +1,32 @@
+{
+ "suite": "MobsterBench-iOS H1",
+ "version": "1.0",
+ "sha256": "2e83875ffbf3c0a3d342320d07ff7632c3e4ecf761926ac7d29e8c2b53cefc76",
+ "frozen_on": "2026-10-09",
+ "task_count": 16,
+ "tranches": {
+  "short": 6,
+  "retrieval": 4,
+  "multi_app": 4,
+  "text_entry": 2
+ },
+ "tasks": [
+  "h.short.battery",
+  "h.short.storage",
+  "h.short.screen_time",
+  "h.short.camera",
+  "h.short.note_link",
+  "h.short.contact_city",
+  "h.ret.calendar_start",
+  "h.ret.reminder_books",
+  "h.ret.turing_birthplace",
+  "h.ret.titanic_year",
+  "h.multi.note_plus",
+  "h.multi.contact_county",
+  "h.multi.room_times",
+  "h.multi.phone_minus",
+  "h.text.maps_bridge",
+  "h.text.settings_wallpaper"
+ ],
+ "note": "Written before any engine idea was built. Never tuned on: anything that learns uses heldout.build_training() and the v1 suite."
+}

mobile_agent/bench/heldout.py

@@ -0,0 +1,375 @@
+"""The held-out tranches (MobsterBench-iOS H1, H2 and H3) and H1's training siblings, each frozen by hash."""
+
+import dataclasses
+
+from .checks import ActedAtLeast, AnswerIs, Claimed, Fixture, InApp, Literal, OnScreen, Truth
+from .suite import (
+    BUNDLES,
+    LOOK_ONLY,
+    NO_CHANGE,
+    READ_ONLY_HISTORY,
+    REVERSIBLE_RESET,
+    SAFARI,
+    SETTINGS,
+    WIKI,
+    Task,
+    _canonical,
+)
+
+HELDOUT_NAME = "MobsterBench-iOS H1"
+HELDOUT_VERSION = "1.0"
+# (max steps, max seconds) per tranche, the same for every arm.
+H_BUDGETS = {"short": (12, 90), "retrieval": (20, 150), "multi_app": (35, 300), "text_entry": (20, 150)}
+
+
+def _h(id, tranche, apps, goal, checks, *, answer=(), **kw):
+    steps, seconds = H_BUDGETS[tranche]
+    return Task(id=id, category=tranche, apps=tuple(apps), goal=goal, checks=tuple(checks), answer=tuple(answer),
+                steps=steps, seconds=seconds, **kw)
+
+
+def _nav(id, title, goal, tranche="short", min_actions=1, **kw):
+    return _h(id, tranche, ["settings"], goal + NO_CHANGE,
+              [Claimed(), ActedAtLeast(min_actions), InApp(SETTINGS), OnScreen(SETTINGS, Literal(title))], **kw)
+
+
+def _ask(id, tranche, apps, goal, field, expect, *, mode="text", alternatives=(), min_actions=1, extra=(), **kw):
+    checks = [Claimed(), ActedAtLeast(min_actions), AnswerIs(field, expect, mode, tuple(alternatives)), *extra]
+    return _h(id, tranche, apps, goal, checks, answer=((field, "string"),), **kw)
+
+
+def build_heldout():
+    return (
+        _nav("h.short.battery", "Battery", "In Settings, open Battery."),
+        _nav("h.short.storage", "iPhone Storage", "In Settings, open General, then iPhone Storage.", min_actions=2),
+        _nav("h.short.screen_time", "Screen Time", "In Settings, open Screen Time."),
+        _nav("h.short.camera", "Camera", "In Settings, open Camera."),
+        _ask("h.short.note_link", "short", ["notes"],
+             "In Notes, open the note titled 'MobsterBench Note' and report the web address written in it."
+             + LOOK_ONLY, "web_address", Literal(r"(https?://)?en\.m\.wikipedia\.org/wiki/alan_turing/?"),
+             mode="regex", fixtures=("notes.bench",)),
+        _ask("h.short.contact_city", "short", ["contacts"],
+             "In Contacts, open the contact Bench Tester and report the city in the address. Do not call, "
+             "message or edit the contact.", "city", Fixture("contacts.bench", "city"),
+             fixtures=("contacts.bench",)),
+
+        _ask("h.ret.calendar_start", "retrieval", ["calendar"],
+             "In Calendar, find the event 'MobsterBench Review' on 15 October 2026 and report its start time. "
+             "Do not edit, delete or respond to any event.", "start_time",
+             Literal(r"(2(:00)? ?p\.?m\.?|14[:.]00)"), mode="regex", fixtures=("calendar.bench",),
+             safety=REVERSIBLE_RESET, allow=("text_entry",)),
+        _ask("h.ret.reminder_books", "retrieval", ["reminders"],
+             "In Reminders, open the list named MobsterBench and report the reminder that mentions books, "
+             "exactly as written. Do not complete, edit or add reminders.", "reminder",
+             Literal("Return library books"), fixtures=("reminders.bench",)),
+        _ask("h.ret.turing_birthplace", "retrieval", ["safari"],
+             "According to this Wikipedia article, in which city was Alan Turing born?", "city",
+             Literal(r"(.*\b)?london(\b.*)?|maida vale.*"), mode="regex", min_actions=0,
+             start_url=WIKI + "Alan_Turing", safety=READ_ONLY_HISTORY, extra=[InApp(SAFARI)]),
+        _ask("h.ret.titanic_year", "retrieval", ["safari"],
+             "According to this Wikipedia article, in what year did the Titanic sink?", "year", Literal("1912"),
+             mode="number", min_actions=0, start_url=WIKI + "Titanic", safety=READ_ONLY_HISTORY,
+             extra=[InApp(SAFARI)]),
+
+        _ask("h.multi.note_plus", "multi_app", ["notes", "calculator"],
+             "Read the locker code in the note 'MobsterBench Note' in Notes, then use Calculator to add 1000 to "
+             "it, and report the result." + LOOK_ONLY, "result", Literal("5817"), mode="number", min_actions=4,
+             fixtures=("notes.bench",), safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+        _ask("h.multi.contact_county", "multi_app", ["contacts", "safari"],
+             "In Contacts, find the city in Bench Tester's address, then in Safari open the English Wikipedia "
+             "article about that city and report the county it is in. Do not call, message or edit the contact.",
+             "county", Literal(r"santa clara( county)?(,? california)?"), mode="regex", min_actions=3,
+             fixtures=("contacts.bench",), safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _ask("h.multi.room_times", "multi_app", ["calendar", "calculator"],
+             "In Calendar, find the location of the event 'MobsterBench Review' on 15 October 2026, then use "
+             "Calculator to multiply the room number in it by 3, and report the result. Do not edit any event.",
+             "result", Literal("126"), mode="number", min_actions=4, fixtures=("calendar.bench",),
+             safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+        _ask("h.multi.phone_minus", "multi_app", ["contacts", "calculator"],
+             "In Contacts, read the last four digits of Bench Tester's phone number, then use Calculator to "
+             "subtract 77 from that four-digit number, and report the result. Do not call, message or edit the "
+             "contact.", "result", Literal("4400"), mode="number", min_actions=4, fixtures=("contacts.bench",),
+             safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+
+        _ask("h.text.maps_bridge", "text_entry", ["maps"],
+             "In Maps, search for the Golden Gate Bridge and report the city it is in. Do not start directions, "
+             "save, share or report anything.", "city", Literal("San Francisco"), alternatives=("SF",),
+             safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _h("h.text.settings_wallpaper", "text_entry", ["settings"],
+           "Use the search field in Settings to find Wallpaper and open the Wallpaper screen." + NO_CHANGE,
+           [Claimed(), ActedAtLeast(2), InApp(SETTINGS), OnScreen(SETTINGS, Literal("Wallpaper"))],
+           safety=REVERSIBLE_RESET, allow=("text_entry",)),
+    )
+
+
+def build_training():
+    """Siblings of the H1 templates (other values), for anything that learns. Never graded as held-out."""
+    return (
+        _nav("t.short.sounds", "Sounds & Haptics", "In Settings, open Sounds & Haptics."),
+        _nav("t.short.airdrop", "AirDrop", "In Settings, open General, then AirDrop.", min_actions=2),
+        _nav("t.short.focus", "Focus", "In Settings, open Focus."),
+        _ask("t.short.note_code", "short", ["notes"],
+             "In Notes, open the note titled 'MobsterBench Note' and report the locker code written in it."
+             + LOOK_ONLY, "code", Fixture("notes.bench", "code"), fixtures=("notes.bench",)),
+        _ask("t.short.contact_company", "short", ["contacts"],
+             "In Contacts, open the contact Bench Tester and report the company name. Do not call, message or "
+             "edit the contact.", "company", Fixture("contacts.bench", "company"), fixtures=("contacts.bench",)),
+        _ask("t.ret.calendar_location", "retrieval", ["calendar"],
+             "In Calendar, find the event 'MobsterBench Review' on 15 October 2026 and report its location. "
+             "Do not edit, delete or respond to any event.", "location", Fixture("calendar.bench", "location"),
+             fixtures=("calendar.bench",), safety=REVERSIBLE_RESET, allow=("text_entry",)),
+        _ask("t.ret.reminder_plants", "retrieval", ["reminders"],
+             "In Reminders, open the list named MobsterBench and report the reminder that mentions plants, "
+             "exactly as written. Do not complete, edit or add reminders.", "reminder",
+             Literal("Water the plants"), fixtures=("reminders.bench",)),
+        _ask("t.ret.hopper_birthplace", "retrieval", ["safari"],
+             "According to this Wikipedia article, in which city was Grace Hopper born?", "city",
+             Literal(r"(.*\b)?new york( city)?(\b.*)?"), mode="regex", min_actions=0,
+             start_url=WIKI + "Grace_Hopper", safety=READ_ONLY_HISTORY),
+        _ask("t.multi.note_minus", "multi_app", ["notes", "calculator"],
+             "Read the locker code in the note 'MobsterBench Note' in Notes, then use Calculator to subtract 800 "
+             "from it, and report the result." + LOOK_ONLY, "result", Literal("4017"), mode="number",
+             min_actions=4, fixtures=("notes.bench",), safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+        _ask("t.multi.room_plus", "multi_app", ["calendar", "calculator"],
+             "In Calendar, find the location of the event 'MobsterBench Review' on 15 October 2026, then use "
+             "Calculator to add 100 to the room number in it, and report the result. Do not edit any event.",
+             "result", Literal("142"), mode="number", min_actions=4, fixtures=("calendar.bench",),
+             safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+        _ask("t.multi.phone_plus", "multi_app", ["contacts", "calculator"],
+             "In Contacts, read the last four digits of Bench Tester's phone number, then use Calculator to add "
+             "23 to that four-digit number, and report the result. Do not call, message or edit the contact.",
+             "result", Literal("4500"), mode="number", min_actions=4, fixtures=("contacts.bench",),
+             safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+        _ask("t.text.maps_liberty", "text_entry", ["maps"],
+             "In Maps, search for the Statue of Liberty and report the city it is in. Do not start directions, "
+             "save, share or report anything.", "city", Literal("New York"),
+             alternatives=("New York City", "NYC", "New York, NY"), safety=READ_ONLY_HISTORY,
+             allow=("text_entry",)),
+    )
+
+
+# The hard set (H-hard, 9 Oct, owner: "harder benchmark problems ... long horizon, complex"): long multi-app chains,
+# web testing and library counts. Not held out: it is for telling architectures apart.
+HARD_BUDGET = (40, 360)
+
+
+def _hard(id, apps, goal, field, expect, *, mode="text", **kw):
+    return Task(id=id, category="hard", apps=tuple(apps), goal=goal,
+                checks=(Claimed(), AnswerIs(field, expect, mode)), answer=((field, "string"),),
+                steps=HARD_BUDGET[0], seconds=HARD_BUDGET[1], **kw)
+
+
+def build_hard():
+    return (
+        _hard("hard.four_apps", ["notes", "reminders", "calendar", "calculator"],
+              "Take the locker code from the note 'MobsterBench Note' in Notes, add the number of incomplete "
+              "reminders in the Reminders list MobsterBench, multiply that sum by the room number in the location "
+              "of the Calendar event 'MobsterBench Review' on 15 October 2026, using Calculator for the arithmetic, "
+              "and report the result. Do not change any note, reminder or event.", "result", Literal("202440"),
+              mode="number", fixtures=("notes.bench", "reminders.bench", "calendar.bench"), safety=REVERSIBLE_RESET,
+              allow=("calculator_keys", "text_entry")),
+        _hard("hard.city_incorporated", ["contacts", "safari"],
+              "In Contacts, find the city in Bench Tester's address. Then in Safari open the English Wikipedia "
+              "article about that city and report the year the city was incorporated. Do not call, message or edit "
+              "the contact.", "year", Literal("1955"), mode="number", fixtures=("contacts.bench",),
+              safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _hard("hard.turing_years", ["notes", "safari", "calculator"],
+              "Open the note 'MobsterBench Note' in Notes and open the web link written in it in Safari. Find the "
+              "year Alan Turing's paper 'On Computable Numbers' was published, then use Calculator to subtract that "
+              "year from 2026, and report the result." + LOOK_ONLY, "result", Literal("90"), mode="number",
+              fixtures=("notes.bench",), safety=REVERSIBLE_RESET, allow=("calculator_keys", "text_entry")),
+        _hard("hard.files_folders", ["files"],
+              "In Files, open On My iPhone, then the MobsterBench folder, and report how many folders it contains."
+              + LOOK_ONLY, "count", Literal("4"), mode="number", fixtures=("files.dogs12",)),
+        _hard("hard.album_count", ["photos"],
+              "In Photos, find the album 'MobsterBench Receipts' and report how many photos it contains." + LOOK_ONLY,
+              "count", Literal("12"), mode="number", fixtures=("photos.receipts12",)),
+        _hard("hard.reminders_sorted", ["reminders"],
+              "In Reminders, open the list MobsterBench and report its incomplete reminders in alphabetical order, "
+              "separated by commas. Do not complete, edit or add reminders.", "reminders",
+              Literal(r"pick up dry cleaning[,;]? (and )?return library books[,;]? (and )?water the plants"),
+              mode="regex", fixtures=("reminders.bench",)),
+        _hard("hard.example_iana", ["safari"],
+              "In Safari, on this page follow the 'More information' link and report the main heading of the page it "
+              "opens.", "heading", Literal("Example Domains"), start_url="https://example.com",
+              safety=READ_ONLY_HISTORY),
+    )
+
+
+COMBO_IDS = ("t.short.sounds", "t.short.contact_company", "t.ret.calendar_location", "t.ret.hopper_birthplace",
+             "t.multi.note_minus", "t.text.maps_liberty")
+
+
+def build_combo():
+    """The small, varied set for reading engine combinations by hand (never H1): six training siblings (short,
+    retrieval, web, multi-app, text entry) and one long v1 task (Settings, then Safari by URL, then a date)."""
+    from .suite import build_suite
+    training = {t.id: t for t in build_training()}
+    return tuple(training[i] for i in COMBO_IDS) + tuple(t for t in build_suite() if t.id == "multi.ios_release")
+
+
+def heldout_document(tasks=None):
+    tasks = build_heldout() if tasks is None else tasks
+    return {"suite": HELDOUT_NAME, "version": HELDOUT_VERSION, "budgets": H_BUDGETS, "bundles": BUNDLES,
+            "tasks": [_canonical(task) for task in tasks]}
+
+
+def heldout_hash(tasks=None):
+    import hashlib
+    import json
+    blob = json.dumps(heldout_document(tasks), sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+    return hashlib.sha256(blob.encode()).hexdigest()
+
+
+def tranche_counts(tasks=None):
+    counts = {}
+    for task in tasks or build_heldout():
+        counts[task.category] = counts.get(task.category, 0) + 1
+    return counts
+
+
+# The harder held-out tranche for the engine-on A/B (owner, 10 Oct): multi-field answers, three-app chains, two-hop
+# web lookups and counts, on the same fixtures and system apps. Written and hashed (h2-manifest.json) before any
+# engine ran it; never tuned on.
+H2_NAME, H2_VERSION = "MobsterBench-iOS H2", "1.0"
+H2_BUDGET = (30, 240)
+
+
+def _h2(id, apps, goal, answers, *, min_actions=1, **kw):
+    """``answers``: ((field, expect, mode), ...), each checked on its own."""
+    checks = [Claimed(), ActedAtLeast(min_actions)] + [AnswerIs(field, expect, mode) for field, expect, mode in answers]
+    return Task(id=id, category="h2", apps=tuple(apps), goal=goal, checks=tuple(checks),
+                answer=tuple((field, "string") for field, _, _ in answers), steps=H2_BUDGET[0], seconds=H2_BUDGET[1],
+                **kw)
+
+
+def build_h2():
+    return (
+        _h2("h2.calendar_three", ["calendar"],
+            "In Calendar, find the event 'MobsterBench Review' on 15 October 2026 and report its start time, its end "
+            "time and the day of the week it falls on. Do not edit, delete or respond to any event.",
+            (("start_time", Literal(r"(2(:00)? ?p\.?m\.?|14[:.]00)"), "regex"),
+             ("end_time", Literal(r"(3(:00)? ?p\.?m\.?|15[:.]00)"), "regex"),
+             ("weekday", Literal("Thursday"), "text")),
+            fixtures=("calendar.bench",), safety=REVERSIBLE_RESET, allow=("text_entry",)),
+        _h2("h2.contact_two", ["contacts"],
+            "In Contacts, open the contact Bench Tester and report the company name and the city in the address. "
+            "Do not call, message or edit the contact.",
+            (("company", Fixture("contacts.bench", "company"), "text"),
+             ("city", Fixture("contacts.bench", "city"), "text")),
+            fixtures=("contacts.bench",)),
+        _h2("h2.about_two", ["settings"],
+            "In Settings > General > About, report the iOS version and the Model Name." + NO_CHANGE,
+            (("ios_version", Truth("device.ios_version"), "text"), ("model_name", Truth("device.model_name"), "text")),
+            min_actions=2),
+        _h2("h2.receipts_count", ["files"],
+            "In Files, open On My iPhone > MobsterBench > receipts12 and report how many files the folder contains."
+            + LOOK_ONLY, (("count", Literal("12"), "number"),), min_actions=2, fixtures=("files.receipts12",)),
+        _h2("h2.room_plus_phone", ["calendar", "contacts", "calculator"],
+            "Find the room number in the location of the Calendar event 'MobsterBench Review' on 15 October 2026, "
+            "then read the last four digits of Bench Tester's phone number in Contacts, then use Calculator to add the "
+            "two numbers, and report the sum. Do not edit, call or message anything.",
+            (("sum", Literal("4519"), "number"),), min_actions=5,
+            fixtures=("calendar.bench", "contacts.bench"), safety=REVERSIBLE_RESET,
+            allow=("calculator_keys", "text_entry")),
+        _h2("h2.code_digit_sum", ["notes", "calculator"],
+            "Read the locker code in the note 'MobsterBench Note' in Notes, then use Calculator to add up its four "
+            "digits, and report the total." + LOOK_ONLY, (("total", Literal("20"), "number"),), min_actions=5,
+            fixtures=("notes.bench",), safety=REVERSIBLE_RESET, allow=("calculator_keys",)),
+        _h2("h2.maps_alcatraz", ["maps"],
+            "In Maps, search for Alcatraz Island and report the city it belongs to. Do not start directions, save, "
+            "share or report anything.", (("city", Literal(r"(.*\b)?san francisco(\b.*)?"), "regex"),),
+            safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _h2("h2.bridge_opened", ["safari"],
+            "According to this Wikipedia article, in what year did the bridge open to the public?",
+            (("year", Literal("1937"), "number"),), min_actions=0, start_url=WIKI + "Golden_Gate_Bridge",
+            safety=READ_ONLY_HISTORY),
+        _h2("h2.turing_phd_university", ["safari"],
+            "From this Wikipedia article, open the article about the university where Alan Turing earned his PhD, and "
+            "report the year that university was founded.", (("founded", Literal("1746"), "number"),),
+            start_url=WIKI + "Alan_Turing", safety=READ_ONLY_HISTORY),
+        _h2("h2.reminder_last_alpha", ["reminders"],
+            "In Reminders, open the list MobsterBench and report which of its incomplete reminders comes last in "
+            "alphabetical order, exactly as written. Do not complete, edit or add reminders.",
+            (("reminder", Literal("Water the plants"), "text"),), fixtures=("reminders.bench",)),
+    )
+
+
+# H3 (10 Oct 2026): a fresh held-out tranche for the default decision (owner: the routed engine becomes the default
+# only when, on a tranche it was never tuned on, its median and p90 per task are at least as fast as Sonnet's with
+# success no worse). Written and hashed (h3-manifest.json) before any engine ran it; H2 is tuned on since 10 Oct.
+H3_NAME, H3_VERSION = "MobsterBench-iOS H3", "1.0"
+H3_BUDGET = H2_BUDGET
+
+
+def _h3(id, apps, goal, answers, **kw):
+    task = _h2(id, apps, goal, answers, **kw)
+    return dataclasses.replace(task, category="h3", steps=H3_BUDGET[0], seconds=H3_BUDGET[1])
+
+
+def build_h3():
+    return (
+        _h3("h3.hopper_retired", ["safari"],
+            "According to this Wikipedia article, in what year did Grace Hopper finally retire from the US Navy?",
+            (("year", Literal("1986"), "number"),), min_actions=0, start_url=WIKI + "Grace_Hopper",
+            safety=READ_ONLY_HISTORY),
+        _h3("h3.lovelace_father", ["safari"],
+            "In Safari, open the English Wikipedia article about Ada Lovelace and report her father's name.",
+            (("father", Literal(r"(.*\b)?byron(\b.*)?"), "regex"),), safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _h3("h3.eiffel_height", ["safari"],
+            "In Safari, open the English Wikipedia article about the Eiffel Tower and report its height in metres "
+            "(the total height, with antennas).", (("height_m", Literal("330"), "number"),), safety=READ_ONLY_HISTORY,
+            allow=("text_entry",)),
+        _h3("h3.apollo_cmp", ["safari"],
+            "According to this Wikipedia article, who was the command module pilot of the mission?",
+            (("pilot", Literal(r"(.*\b)?collins(\b.*)?"), "regex"),), min_actions=0, start_url=WIKI + "Apollo_11",
+            safety=READ_ONLY_HISTORY),
+        _h3("h3.python_designer", ["safari"],
+            "In Safari, open the English Wikipedia article about the Python programming language and report who "
+            "designed it.", (("designer", Literal(r"(.*\b)?guido van rossum(\b.*)?"), "regex"),),
+            safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _h3("h3.coit_city", ["maps"],
+            "In Maps, search for Coit Tower and report the city it is in. Do not start directions, save, share or "
+            "report anything.", (("city", Literal(r"(.*\b)?san francisco(\b.*)?"), "regex"),),
+            safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _h3("h3.space_needle_two", ["maps", "safari"],
+            "In Maps, search for the Space Needle and report the city it is in, then in Safari open the English "
+            "Wikipedia article about the Space Needle and report the year it opened. Do not start directions, save "
+            "or share anything.", (("city", Literal(r"(.*\b)?seattle(\b.*)?"), "regex"),
+                                    ("opened", Literal("1962"), "number")),
+            min_actions=3, safety=READ_ONLY_HISTORY, allow=("text_entry",)),
+        _h3("h3.sunglasses_count", ["files"],
+            "In Files, open On My iPhone > MobsterBench > sunglasses12 and report how many files the folder contains."
+            + LOOK_ONLY, (("count", Literal("12"), "number"),), min_actions=2, fixtures=("files.sunglasses12",)),
+        _h3("h3.locker_and_room", ["notes", "calendar"],
+            "Report the locker code written in the note 'MobsterBench Note' in Notes, and the room number in the "
+            "location of the Calendar event 'MobsterBench Review' on 15 October 2026. Do not edit anything.",
+            (("locker_code", Literal("4817"), "number"), ("room", Literal("42"), "number")), min_actions=3,
+            fixtures=("notes.bench", "calendar.bench"), safety=REVERSIBLE_RESET, allow=("text_entry",)),
+        _h3("h3.phone_minus_room", ["contacts", "calendar", "calculator"],
+            "Read the last four digits of Bench Tester's phone number in Contacts and the room number in the location "
+            "of the Calendar event 'MobsterBench Review' on 15 October 2026, then use Calculator to subtract the room "
+            "number from the four digits, and report the result. Do not edit, call or message anything.",
+            (("result", Literal("4435"), "number"),), min_actions=5, fixtures=("contacts.bench", "calendar.bench"),
+            safety=REVERSIBLE_RESET, allow=("calculator_keys", "text_entry")),
+    )
+
+
+def h3_hash(tasks=None):
+    import hashlib
+    import json
+    tasks = build_h3() if tasks is None else tasks
+    blob = json.dumps({"suite": H3_NAME, "version": H3_VERSION, "budget": H3_BUDGET, "bundles": BUNDLES,
+                       "tasks": [_canonical(t) for t in tasks]}, sort_keys=True, separators=(",", ":"),
+                      ensure_ascii=True)
+    return hashlib.sha256(blob.encode()).hexdigest()
+
+
+def h2_hash(tasks=None):
+    import hashlib
+    import json
+    tasks = build_h2() if tasks is None else tasks
+    blob = json.dumps({"suite": H2_NAME, "version": H2_VERSION, "budget": H2_BUDGET, "bundles": BUNDLES,
+                       "tasks": [_canonical(t) for t in tasks]}, sort_keys=True, separators=(",", ":"),
+                      ensure_ascii=True)
+    return hashlib.sha256(blob.encode()).hexdigest()