mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/bench/tape_replay.py

@@ -0,0 +1,88 @@
+"""Replay recorded runs on today's code: every action the same, or the first one that isn't. $0, no simulator.
+
+    python -m mobile_agent.bench.tape_replay research/iosworld-runs/RUN ...     every tape.jsonl under each folder
+    python -m mobile_agent.bench.tape_replay mobster-trace-3f9c2a1b7d10.zip  a trace bundle (mobster export ID --trace)
+    python -m mobile_agent.bench.tape_replay DIR --steps                          each recorded action, same or differs
+    python -m mobile_agent.bench.tape_replay DIR --json                           one JSON report per tape
+
+Each recorded run (bench/iosworld.py writes tape.jsonl beside prompts.jsonl) is run again through FrontierAgent with
+the recorded phone and model answering (tape.py). A run is **identical** when the replay sends the same actions in the
+same order, with the same targets and text, and ends with the same status. A **divergence** names the first action
+that differs and the turn it came in: the place a change to the loop, its gates or its target resolution would act
+differently on that run. **Prompts same** counts the model calls whose prompt text is byte for byte the recording's.
+
+The exit status is 1 when any tape diverges or errors, so a gate can run it.
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import tape
+
+
+def tapes(paths):
+    found = []
+    for path in map(Path, paths):
+        names = (tape.NAME, tape.NAME + ".gz")
+        if path.is_file():
+            found.append(path)
+        elif any((path / name).is_file() for name in names):
+            found.append(next(path / name for name in names if (path / name).is_file()))
+        else:
+            # Run folders hold tape.jsonl(.gz); a folder of saved tapes (tests/fixtures/tapes) holds NAME.jsonl.gz.
+            found.extend(sorted(p for name in names for p in path.rglob(name))
+                         or sorted([*path.glob("*.jsonl.gz"), *path.glob("*.zip")]))
+    return found
+
+
+def line(report):
+    path = Path(report.tape)
+    where = path.parent.name if path.name.startswith(tape.NAME) else path.name.split(".")[0]
+    if report.error:
+        verdict = f"error: {report.error[:90]}"
+    elif report.divergence:
+        d = report.divergence
+        verdict = (f"DIVERGES at action {d['index']} (turn {d['step']}): {d['why']}; recorded "
+                   f"{json.dumps(d['expected'])[:80]}, now {json.dumps(d['got'])[:80]}")
+    elif report.identical:
+        verdict = "identical"
+    else:
+        verdict = f"ends differently: {report.recorded_status} -> {report.replayed_status}"
+    return (f"{where[:40]:40}  {report.replayed_acts:3}/{report.recorded_acts:<3} acts  "
+            f"prompts {report.prompts_same}/{report.prompts_seen}  {report.seconds:5.2f}s  {verdict}")
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.tape_replay",
+                                     description=__doc__.split("\n")[0])
+    parser.add_argument("paths", nargs="+",
+                        help="tape.jsonl files, trace bundles (.zip), or folders to search for them")
+    parser.add_argument("--json", action="store_true", help="one JSON report a line")
+    parser.add_argument("--steps", action="store_true", help="under each run, every recorded action: same, or differs")
+    args = parser.parse_args(argv)
+    found = tapes(args.paths)
+    if not found:
+        print("No tape.jsonl or tape.jsonl.gz under " + ", ".join(args.paths), file=sys.stderr)
+        return 2
+    reports = [tape.replay(path) for path in found]
+    for report in reports:
+        print(json.dumps(report.public(), ensure_ascii=False) if args.json else line(report))
+        if args.steps and not args.json:
+            if report.hooks:
+                print(f"    the live run also had {', '.join(report.hooks)}; a replay has none of them")
+            for step in report.steps:
+                print(f"    {step['index'] + 1:3}. turn {step['turn'] or '?':>3}  {step['act'][:70]:70}  {step['now']}")
+    same = sum(r.identical for r in reports)
+    acts = sum(r.recorded_acts for r in reports)
+    replayed = sum(min(r.replayed_acts, r.recorded_acts) for r in reports)
+    prompts = sum(r.prompts_same for r in reports), sum(r.prompts_seen for r in reports)
+    if not args.json:
+        print(f"\n{same} of {len(reports)} runs identical; {replayed} of {acts} recorded actions replayed; "
+              f"prompts byte for byte the same: {prompts[0]} of {prompts[1]}")
+    return 0 if same == len(reports) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mobile_agent/bench/trace_evals.py

@@ -0,0 +1,128 @@
+"""The local trace evals: runs kept as regression tasks, replayed offline against today's code. No phone, no network,
+no model.
+
+    python -m mobile_agent.bench evals                      the app's eval set, beside its journal
+    python -m mobile_agent.bench evals --dir DIR --json     another set, one JSON object a task
+
+A task is a folder in the eval set (`mobster history --to-eval ID` and the app's Add to evals write it):
+
+    <id>/task.json        {"format", "id", "added", "goal", "app", "source": {"run", "version"},
+                           "recorded": {"status", "outcome", "steps", "actions", "costUsd", "failure"},
+                           "expected": {"status", "replay"}}
+    <id>/tape.jsonl.gz    the run's tape (bench/tape.py), masked
+
+``expected.status`` is the outcome a fixed engine should reach: "completed" for a task that failed. Only a live run
+can show it, so the offline check asks something smaller, ``expected.replay``:
+
+- ``changes``: the task failed. Today's code replaying it either acts the same way (**open**: the failure is still
+  there) or acts differently at some step (**changed**: a candidate fix, to be confirmed by a paired live run).
+- ``holds``: the task did not fail. Today's code must replay it with every action the same, or it is **broken**.
+
+A task that can't be read or replayed is an **error**. The exit status is 1 when any task is broken or errors.
+"""
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+from . import tape
+
+FORMAT = "mobster-trace-eval/1"
+TASK, TAPE = "task.json", tape.NAME + ".gz"
+REPLAYS = ("changes", "holds")
+ID = re.compile(r"[a-z0-9][a-z0-9-]{0,80}")
+
+
+class BadTask(ValueError):
+    pass
+
+
+def read_task(folder):
+    """The task in ``folder``, validated; BadTask says what is wrong with it."""
+    folder = Path(folder)
+    try:
+        task = json.loads((folder / TASK).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        raise BadTask(f"{folder.name}: no readable {TASK}") from None
+    expected = task.get("expected") if isinstance(task, dict) else None
+    if (not isinstance(task, dict) or task.get("format") != FORMAT or task.get("id") != folder.name
+            or not ID.fullmatch(folder.name) or not isinstance(expected, dict)
+            or expected.get("replay") not in REPLAYS or not isinstance(task.get("recorded"), dict)):
+        raise BadTask(f"{folder.name}: {TASK} is not a {FORMAT} task for this folder")
+    if not (folder / TAPE).is_file():
+        raise BadTask(f"{folder.name}: no {TAPE}")
+    return task
+
+
+def judge(task, report):
+    """(verdict, detail) for one replay ``report`` (tape.Report) of ``task``: holds, broken, open, changed or error."""
+    if report.error:
+        return "error", report.error
+    where = ""
+    if report.divergence:
+        step = report.divergence
+        where = f"at action {step['index'] + 1} (turn {step['step']}): {step['why']}"
+    elif not report.identical:
+        where = f"ends {report.replayed_status} where the recording ended {report.recorded_status}"
+    if task["expected"]["replay"] == "holds":
+        return ("holds", "every action the same") if report.identical else ("broken", where)
+    return ("open", "acts as it did when it failed") if report.identical else ("changed", where)
+
+
+def run(root):
+    """[{id, verdict, detail, acts, seconds}] for every task folder under ``root`` (an unreadable one is an error
+    row)."""
+    root = Path(root)
+    rows = []
+    folders = root.iterdir() if root.is_dir() else ()
+    for folder in sorted(p for p in folders if p.is_dir() and not p.name.startswith(".")):
+        try:
+            task = read_task(folder)
+        except BadTask as error:
+            rows.append({"id": folder.name, "verdict": "error", "detail": str(error), "acts": None, "seconds": 0})
+            continue
+        report = tape.replay(folder / TAPE)
+        verdict, detail = judge(task, report)
+        rows.append({"id": task["id"], "verdict": verdict, "detail": detail, "goal": task.get("goal"),
+                     "acts": f"{report.replayed_acts}/{report.recorded_acts}", "seconds": report.seconds,
+                     "expected": task["expected"]["replay"]})
+    return rows
+
+
+def add_arguments(parser):
+    parser.add_argument("--dir", type=Path, help="the eval set's folder (default: the app's, beside its journal)")
+    parser.add_argument("--json", action="store_true", help="one JSON object a task")
+
+
+def command(args):
+    root = args.dir
+    if root is None:
+        from ..paths import default_history_db
+        root = default_history_db().parent / "evals"
+    rows = run(root)
+    if not rows:
+        print(f"No trace evals in {root}. Add a finished task with `mobster history --to-eval ID`.", file=sys.stderr)
+        return 2
+    for row in rows:
+        if args.json:
+            print(json.dumps(row, ensure_ascii=False))
+        else:
+            acts = f"{row['acts']} acts" if row["acts"] else "-"
+            print(f"{row['id'][:48]:48}  {row['verdict']:7}  {acts:>11}  {row['seconds']:5.2f}s  {row['detail']}")
+    counts = {verdict: sum(row["verdict"] == verdict for row in rows) for verdict in
+              ("open", "changed", "holds", "broken", "error")}
+    if not args.json:
+        print(f"\n{len(rows)} tasks: " + ", ".join(f"{n} {name}" for name, n in counts.items() if n))
+    return 1 if counts["broken"] or counts["error"] else 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench evals", description=__doc__.split("\n")[0])
+    add_arguments(parser)
+    return command(parser.parse_args(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mobile_agent/bench/trace_view.py

@@ -0,0 +1,81 @@
+"""A run's steps as a table, for reading engine combinations by hand.
+
+  python -m mobile_agent.bench.trace_view RUN_DIR [--task ID]
+
+Per attempt: one line per decision with its latency and cost, the actions it took and whether each changed the
+screen, refusals, chunk stops, and the end: status, answer, verdict and the failing checks.
+"""
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+def load(path):
+    try:
+        return [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]
+    except OSError:
+        return []
+
+
+def attempt_lines(folder, record):
+    events = load(folder / "events.jsonl")
+    prompts = {p.get("step"): p for p in load(folder / "prompts.jsonl")}
+    lines = [f"## {record['task']} [{record['agent']}] verdict={record['verdict']} status={record.get('status')} "
+             f"{(record.get('agent_ms') or 0) / 1000:.1f}s calls={record.get('model_calls')} "
+             f"turns={record.get('decision_steps')} actions={record.get('action_count')} "
+             f"${record.get('cost_usd') or 0:.4f} {record.get('error') or ''}"]
+    pending_cost, pending_ms = 0.0, []
+    for event in events:
+        kind = event.get("event")
+        if kind == "inference_finished":
+            pending_cost += event.get("estimated_usd") or 0
+            pending_ms.append(f"{event.get('purpose')}:{(event.get('latency_ms') or 0) / 1000:.2f}s")
+        elif kind == "frontier_decision":
+            out = (prompts.get(event.get("step")) or {}).get("out") or {}
+            ops = " | ".join(f"{a.get('operation')} {a.get('target') or a.get('target_label') or a.get('app') or ''}"
+                             + (f" {a.get('text')!r}" if a.get("text") else "") for a in out.get("actions") or ())
+            lines.append(f"- t={event['t_ms'] / 1000:6.1f} step {event.get('step')} "
+                         f"[{event.get('source') or 'planner'}] {', '.join(pending_ms)} ${pending_cost:.4f} :: "
+                         f"{event.get('thought', '')[:90]!r} :: {ops[:160]}"
+                         + (f" :: next={event.get('next')}" if event.get("next") else ""))
+            pending_cost, pending_ms = 0.0, []
+        elif kind == "frontier_action":
+            lines.append(f"    {event.get('operation')} {str(event.get('target_label'))[:50]!r} "
+                         f"changed={event.get('changed')} exec={event.get('exec_ms')}ms "
+                         f"settle={event.get('settle_ms')}ms")
+        elif kind in ("frontier_refused", "frontier_chunk_stop", "frontier_failed"):
+            lines.append(f"    ! {kind} {event.get('label') or event.get('reason') or event.get('error') or ''} "
+                         f"{str(event.get('wanted') or event.get('detail') or '')[:80]}")
+        elif kind == "frontier_contract":
+            lines.append(f"    contract: {event.get('items')}")
+    summary = {}
+    try:
+        summary = json.loads((folder / "summary.json").read_text())
+    except (OSError, ValueError):
+        pass
+    failing = "; ".join(c["detail"] for c in record.get("checks", []) if c.get("ok") is False)
+    lines.append(f"- END answer={str(record.get('answer'))[:120]} outcome={summary.get('outcome')} "
+                 f"full={str(summary.get('full_answer') or summary.get('answer'))[:160]!r} {failing}")
+    if record.get("detail", {}).get("timing"):
+        lines.append(f"- timing {record['detail']['timing']}")
+    return lines
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.trace_view")
+    parser.add_argument("run")
+    parser.add_argument("--task")
+    args = parser.parse_args(argv)
+    run = Path(args.run)
+    for record in load(run / "records.jsonl"):
+        if args.task and record["task"] != args.task:
+            continue
+        folder = run / "artifacts" / re.sub(r"[^\w.-]", "_", record["key"])
+        print("\n".join(attempt_lines(folder, record)) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mobile_agent/bench/truth/iphone15pro.json

@@ -0,0 +1,138 @@
+{
+ "device": "iphone15pro",
+ "values": {
+  "device.capacity": {
+   "recorded_at": "2026-09-24T10:50:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "128 GB"
+  },
+  "device.ios_major_release_date": {
+   "source": "seed: public fact (iOS 26 released 15 September 2025); regex",
+   "value": "(september 15,? 2025|15 september 2025|2025-09-15)"
+  },
+  "device.ios_version": {
+   "recorded_at": "2026-09-24T10:50:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "26.0.1"
+  },
+  "device.model_name": {
+   "recorded_at": "2026-09-24T10:50:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "iPhone 15 Pro"
+  },
+  "device.model_number": {
+   "recorded_at": "2026-09-24T10:50:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "MTQM3LL/A"
+  },
+  "device.model_release_year": {
+   "source": "seed: public fact (iPhone 15 Pro released September 2023)",
+   "value": "2023"
+  },
+  "device.modem_firmware": {
+   "recorded_at": "2026-09-24T10:50:21+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "3.02.03"
+  },
+  "settings.auto_correction": {
+   "recorded_at": "2026-09-24T10:49:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "off"
+  },
+  "settings.auto_lock": {
+   "recorded_at": "2026-09-24T07:24:15+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Never"
+  },
+  "settings.bold_text": {
+   "recorded_at": "2026-09-24T10:52:22+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "on"
+  },
+  "settings.low_power": {
+   "recorded_at": "2026-09-24T10:52:02+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "off"
+  },
+  "settings.region": {
+   "recorded_at": "2026-09-24T10:52:13+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "United States"
+  },
+  "settings.time_auto": {
+   "recorded_at": "2026-09-24T10:50:10+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "on"
+  },
+  "title.about": {
+   "recorded_at": "2026-09-24T10:50:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "About"
+  },
+  "title.accessibility": {
+   "recorded_at": "2026-09-24T10:49:08+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Accessibility"
+  },
+  "title.apps": {
+   "recorded_at": "2026-09-24T10:50:52+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Apps"
+  },
+  "title.auto_lock": {
+   "recorded_at": "2026-09-24T07:26:12+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Auto-Lock"
+  },
+  "title.date_time": {
+   "recorded_at": "2026-09-24T10:50:10+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Date & Time"
+  },
+  "title.display": {
+   "recorded_at": "2026-09-24T10:49:42+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Display & Brightness"
+  },
+  "title.files_on_my_iphone": {
+   "recorded_at": "2026-09-24T10:51:53+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "On My iPhone"
+  },
+  "title.formats": {
+   "recorded_at": "2026-09-24T10:50:30+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Formats"
+  },
+  "title.keyboards": {
+   "recorded_at": "2026-09-24T10:49:19+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Keyboards"
+  },
+  "title.legal": {
+   "recorded_at": "2026-09-24T10:51:05+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Legal & Regulatory"
+  },
+  "title.location_services": {
+   "recorded_at": "2026-09-24T10:49:33+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Location Services"
+  },
+  "title.notifications": {
+   "recorded_at": "2026-09-24T10:50:00+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Notifications"
+  },
+  "title.per_app": {
+   "recorded_at": "2026-09-24T10:51:21+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Per-App Settings"
+  },
+  "title.privacy": {
+   "recorded_at": "2026-09-24T10:50:41+00:00",
+   "source": "capture: read on device 2026-09-24",
+   "value": "Privacy & Security"
+  }
+ }
+}