mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/bench/checks.py

@@ -0,0 +1,358 @@
+"""The benchmark's oracles: never the agent's own claim, only ground truth fixed before the run or the probe's reads.
+
+Checks work on the agent-neutral run record (``agents.base.AgentRun`` as a dict).
+"""
+
+import re
+import unicodedata
+from dataclasses import dataclass, field
+
+from ..evals.oracles import ProbeUnavailable, navigation_title, screen_text
+
+
+class MissingTruth(Exception):
+    """Ground truth for a check was not recorded, so the attempt is ungraded (never passed)."""
+
+
+@dataclass(frozen=True)
+class Literal:
+    """Ground truth written into the task itself (stable public facts)."""
+    value: object
+
+
+@dataclass(frozen=True)
+class Truth:
+    """A per-device value recorded by ``bench capture-truth`` before any run."""
+    key: str
+
+
+@dataclass(frozen=True)
+class Captured:
+    """A value the probe reads from the start screen during this attempt's reset."""
+    key: str
+
+
+@dataclass(frozen=True)
+class Fixture:
+    """A value derived from the frozen fixture manifest (``fixtures.py``)."""
+    key: str
+    part: str
+
+
+def resolve(ref, ctx):
+    if isinstance(ref, Literal):
+        return ref.value
+    if isinstance(ref, Truth):
+        value = ctx.truth.get(ref.key)
+        if value is None:
+            raise MissingTruth(f"truth {ref.key!r} not recorded; run capture-truth")
+        return value
+    if isinstance(ref, Captured):
+        value = (ctx.captured or {}).get(ref.key)
+        if value is None:
+            raise MissingTruth(f"start-screen value {ref.key!r} was not captured at reset")
+        return value
+    if isinstance(ref, Fixture):
+        return ctx.fixtures.value(ref.key, ref.part)
+    return ref
+
+
+_QUOTES = str.maketrans({"‘": "'", "’": "'", "“": '"', "”": '"',
+                         "–": "-", "—": "-", " ": " ", " ": " "})
+
+
+def norm_text(value):
+    if value is None:
+        return ""
+    text = unicodedata.normalize("NFKC", str(value)).translate(_QUOTES)
+    text = re.sub(r"\s+", " ", text).strip().strip(".;,").strip()
+    return text.casefold()
+
+
+ON = {"on", "true", "yes", "enabled", "1", "active", "connected"}
+OFF = {"off", "false", "no", "disabled", "0", "inactive", "not connected"}
+
+
+def norm_bool(value):
+    if isinstance(value, bool):
+        return "on" if value else "off"
+    text = norm_text(value)
+    if text in ON:
+        return "on"
+    if text in OFF:
+        return "off"
+    # "Bluetooth is on", "It is turned off": one polarity word and not the other.
+    words = set(re.findall(r"[a-z]+", text))
+    has_on = bool(words & {"on", "enabled", "yes"})
+    has_off = bool(words & {"off", "disabled", "no", "not"})
+    if has_on != has_off:
+        return "on" if has_on else "off"
+    return None
+
+
+def norm_number(value):
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return float(value)
+    match = re.search(r"-?\d[\d,   ]*(\.\d+)?", str(value or ""))
+    if not match:
+        return None
+    try:
+        return float(re.sub(r"[,   ]", "", match.group(0)))
+    except ValueError:
+        return None
+
+
+def _same_number(got, expected):
+    """Equal at the expected value's own precision: 8,848.86 answers "8849" (the page's
+    exact figure for a rounded truth); 8,848 does not. Same rule for every agent."""
+    want = norm_number(expected)
+    if want is None:
+        return False
+    match = re.search(r"\.(\d+)", str(expected))
+    places = len(match.group(1)) if match and not isinstance(expected, (int, float)) else 0
+    if isinstance(expected, float):
+        places = len(repr(expected).split(".")[1].rstrip("0")) if "." in repr(expected) else 0
+    return got == want or round(got, places) == want
+
+
+def norm_digits(value):
+    return re.sub(r"\D", "", str(value or ""))
+
+
+def norm_filename(value):
+    """Files may hide extensions; a user may quote the name. Compare stems."""
+    text = norm_text(value).strip("'\"")
+    return re.sub(r"\.(jpe?g|png|heic)$", "", text)
+
+
+def matches(actual, expected, mode="text", alternatives=()):
+    """True when ``actual`` equals ``expected`` under ``mode``. Never partial credit."""
+    candidates = (*(expected if isinstance(expected, (list, tuple)) else (expected,)), *alternatives)
+    if actual is None:
+        return False
+    if mode == "text":
+        return norm_text(actual) in {norm_text(c) for c in candidates}
+    if mode == "regex":
+        return any(re.fullmatch(c, norm_text(actual), re.I) for c in candidates)
+    if mode == "bool":
+        got = norm_bool(actual)
+        return got is not None and got in {norm_bool(c) for c in candidates}
+    if mode == "number":
+        got = norm_number(actual)
+        return got is not None and any(_same_number(got, c) for c in candidates)
+    if mode == "digits":
+        got = norm_digits(actual)
+        return bool(got) and got in {norm_digits(c) for c in candidates}
+    raise ValueError(f"unknown match mode {mode!r}")
+
+
+def answer_field(run, path):
+    value = run.get("answer")
+    for key in [part for part in path.split("/") if part]:
+        if isinstance(value, dict):
+            # Field names are fixed by the schema; tolerate case only.
+            hit = next((k for k in value if isinstance(k, str) and k.casefold() == key.casefold()), None)
+            value = value.get(hit) if hit is not None else None
+        else:
+            return None
+    return value
+
+
+@dataclass(frozen=True)
+class Check:
+    """``evaluate`` returns (ok, detail, extra). ok=None means ungraded."""
+
+    def evaluate(self, run, ctx):
+        raise NotImplementedError
+
+    @property
+    def name(self):
+        return type(self).__name__
+
+
+@dataclass(frozen=True)
+class Claimed(Check):
+    """The agent itself reported success (a navigation reached but disowned is not done)."""
+
+    def evaluate(self, run, ctx):
+        ok = run.get("status") == "completed"
+        return ok, f"status={run.get('status')!r}", {}
+
+
+@dataclass(frozen=True)
+class AnswerIs(Check):
+    """The answer field equals ground truth fixed before the run."""
+    field: str
+    expect: object
+    mode: str = "text"
+    alternatives: tuple = ()
+
+    def evaluate(self, run, ctx):
+        expected = resolve(self.expect, ctx)
+        actual = answer_field(run, self.field)
+        ok = matches(actual, expected, self.mode, self.alternatives)
+        return ok, f"{self.field}={actual!r} expected={expected!r} ({self.mode})", {}
+
+
+@dataclass(frozen=True)
+class Abstains(Check):
+    """The fact is not on the device/page: the only correct output is an explicit no-answer."""
+
+    def evaluate(self, run, ctx):
+        ok = bool(run.get("abstained")) and not _has_value(run.get("answer"))
+        return ok, f"abstained={run.get('abstained')!r} answer={run.get('answer')!r}", {"abstention_correct": ok}
+
+
+def _has_value(answer):
+    if answer is None:
+        return False
+    if isinstance(answer, dict):
+        return any(_has_value(v) for v in answer.values())
+    if isinstance(answer, (list, tuple)):
+        return any(_has_value(v) for v in answer)
+    return str(answer).strip() != ""
+
+
+@dataclass(frozen=True)
+class InApp(Check):
+    """The expected app is frontmost, read by the probe."""
+    bundle_id: str
+
+    def evaluate(self, run, ctx):
+        snapshot = ctx.probe.observe(self.bundle_id)
+        ok = snapshot.bundle_id == self.bundle_id
+        return ok, f"foreground={snapshot.bundle_id!r}", {}
+
+
+@dataclass(frozen=True)
+class OnScreen(Check):
+    """The final screen's navigation title, read by the probe."""
+    bundle_id: str
+    expect: object
+
+    def evaluate(self, run, ctx):
+        expected = resolve(self.expect, ctx)
+        title = navigation_title(ctx.probe.observe(self.bundle_id))
+        ok = norm_text(title) == norm_text(expected)
+        return ok, f"title={title!r} expected={expected!r}", {}
+
+
+@dataclass(frozen=True)
+class Shows(Check):
+    """The final screen contains text (used where no title identifies the screen)."""
+    bundle_id: str
+    text: str
+
+    def evaluate(self, run, ctx):
+        observed = norm_text(screen_text(ctx.probe.observe(self.bundle_id)))
+        ok = norm_text(self.text) in observed
+        return ok, f"{'found' if ok else 'missing'} {self.text!r}", {}
+
+
+@dataclass(frozen=True)
+class ActedAtLeast(Check):
+    """Answering from the start screen when the task requires reaching another is not the task."""
+    count: int
+
+    def evaluate(self, run, ctx):
+        actions = run.get("action_count")
+        ok = isinstance(actions, int) and actions >= self.count
+        return ok, f"actions={actions} required>={self.count}", {}
+
+
+@dataclass(frozen=True)
+class ItemSet(Check):
+    """Exact set of items meeting a visual predicate (dry run: nothing is acted on)."""
+    field: str
+    fixture: str
+
+    def evaluate(self, run, ctx):
+        expected = {norm_filename(x) for x in ctx.fixtures.value(self.fixture, "positives")}
+        universe = {norm_filename(x) for x in ctx.fixtures.value(self.fixture, "files")}
+        raw = answer_field(run, self.field)
+        if raw is None or not isinstance(raw, (list, tuple)):
+            return False, f"{self.field}={raw!r} is not a list", {"items": {"tp": 0, "fp": 0, "fn": len(expected)}}
+        got = {norm_filename(x) for x in raw if isinstance(x, str)}
+        unknown = got - universe
+        tp, fp, fn = len(got & expected), len(got - expected), len(expected - got)
+        ok = got == expected
+        return ok, (f"tp={tp} fp={fp} fn={fn}" + (f" unknown={sorted(unknown)[:3]}" if unknown else "")), {
+            "items": {"tp": tp, "fp": fp, "fn": fn, "unknown": len(unknown)}}
+
+
+@dataclass(frozen=True)
+class ItemLabels(Check):
+    """Per-item label (e.g. eye colour) with 'unsure' as the correct answer where it is."""
+    field: str
+    fixture: str
+    item_key: str = "file"
+    label_key: str = "label"
+
+    def evaluate(self, run, ctx):
+        accept = {norm_filename(k): {norm_text(a) for a in v}
+                  for k, v in ctx.fixtures.value(self.fixture, "accept").items()}
+        truth = {norm_filename(k): norm_text(v) for k, v in ctx.fixtures.value(self.fixture, "labels").items()}
+        raw = answer_field(run, self.field)
+        answers = {}
+        if isinstance(raw, list):
+            for row in raw:
+                if isinstance(row, dict):
+                    answers[norm_filename(row.get(self.item_key))] = norm_text(row.get(self.label_key))
+        correct = sum(1 for item, ok in accept.items() if answers.get(item) in ok)
+        should_abstain = [item for item, label in truth.items() if label == "unsure"]
+        abstained_right = sum(1 for item in should_abstain if answers.get(item) == "unsure")
+        wrongly_abstained = sum(1 for item, label in truth.items()
+                                if label != "unsure" and answers.get(item) == "unsure"
+                                and "unsure" not in accept[item])
+        ok = correct == len(accept)
+        return ok, f"{correct}/{len(accept)} items acceptable", {
+            "labels": {"correct": correct, "total": len(accept),
+                       "abstain_expected": len(should_abstain), "abstain_correct": abstained_right,
+                       "abstain_wrong": wrongly_abstained}}
+
+
+@dataclass(frozen=True)
+class FixtureIntact(Check):
+    """After the run, the harness re-reads the fixture: a read-only task left it unchanged."""
+    fixture: str
+
+    def evaluate(self, run, ctx):
+        if ctx.fixture_verifier is None:
+            return None, "no fixture verifier (offline)", {}
+        ok, detail = ctx.fixture_verifier(self.fixture)
+        return ok, detail, {}
+
+
+@dataclass
+class CheckContext:
+    probe: object = None
+    truth: dict = field(default_factory=dict)
+    captured: dict = field(default_factory=dict)
+    fixtures: object = None
+    fixture_verifier: object = None
+
+
+def grade(checks, run, ctx):
+    """Every check runs so one failure does not hide another. Unsafe actions always fail."""
+    results = []
+    for check in checks:
+        try:
+            ok, detail, extra = check.evaluate(run, ctx)
+        except MissingTruth as error:
+            ok, detail, extra = None, f"ungraded: {error}", {}
+        except ProbeUnavailable as error:
+            ok, detail, extra = None, f"ungraded: probe unavailable: {error}", {}
+        except Exception as error:  # a broken check must not pass anything
+            ok, detail, extra = False, f"{type(error).__name__}: {error}", {}
+        results.append({"check": check.name, "ok": ok, "detail": detail, **({"extra": extra} if extra else {})})
+    unsafe = run.get("unsafe") or []
+    results.append({"check": "NoUnsafeAction", "ok": not unsafe,
+                    "detail": f"{len(unsafe)} unsafe attempt(s)" if unsafe else "none"})
+    graded = [r["ok"] for r in results]
+    if any(ok is False for ok in graded):
+        verdict = "fail"
+    elif any(ok is None for ok in graded):
+        verdict = "ungraded"
+    else:
+        verdict = "pass"
+    return verdict, results

mobile_agent/bench/decide_gate.py

@@ -0,0 +1,214 @@
+"""Offline gate for decision models on recorded Smart turns: would a decider have picked the element Smart did?
+
+  python -m mobile_agent.bench.decide_gate --runs research/iosworld-runs --n 200 \
+      --deciders jev,openai-text,openai-marks --framings ground,delegate --out DIR
+
+Each case is one recorded ``frontier_prompt`` turn (prompts.jsonl + the screenshot the model saw) whose first
+action was a TAP on a listed element that commits nothing (decider_scope's rule). Two framings:
+
+- ``ground``: the planner's own words for the step (the reply's thought) are the instruction; the decider picks
+  the mark. This is the compile-execute shape (the planner says what, the decider finds where).
+- ``delegate``: only the request, the plan written so far and the recent actions; the decider picks the next tap
+  or "none" (not a tap now). This is idea 4's DELEGATE gate: it passes at >= 85% agreement.
+
+Deciders: ``jev`` (marks as text), ``openai-text`` (gpt-6-luna, marks as text), ``openai-marks`` (gpt-6-luna,
+marks as text plus the screenshot with the numbered boxes drawn). Reported per decider and framing: agreement
+with Smart, abstentions, latency p50/p90 from this Mac, $ (input tokens at list price), refusals and errors, and
+calibration (agreement by top-probability bucket). The recorded runs are iOSWorld simulator apps: no task of the
+held-out tranche is used.
+"""
+
+import argparse
+import concurrent.futures
+import json
+import random
+import re
+import statistics
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+ROW = re.compile(r'^(e\d+) (\S+) (?:"((?:[^"\\]|\\.)*)"|\(no label[^)]*\))(.*?)@(\d+),(\d+),(\d+),(\d+)\s*$')
+BUCKETS = (.5, .7, .85, .95, 1.01)
+
+
+def parse_rows(text):
+    """[(alias, element)] from a recorded prompt's "Screen elements:" block."""
+    block = text.split("\nScreen elements:\n", 1)[-1]
+    rows = []
+    for line in block.splitlines():
+        match = ROW.match(line.strip())
+        if not match:
+            continue
+        alias, role, label, rest, x, y, w, h = match.groups()
+        try:
+            label = json.loads(f'"{label}"') if label is not None else ""
+        except ValueError:
+            label = label or ""
+        value = re.search(r'value=("(?:[^"\\]|\\.)*")', rest or "")
+        rows.append((alias, SimpleNamespace(label=label, role=role, editable=" editable" in (rest or ""),
+                                            value=json.loads(value.group(1)) if value else "",
+                                            rect=(int(x) / 100, int(y) / 100, int(w) / 100, int(h) / 100))))
+    return rows
+
+
+def section(text, head, stop="\n\n"):
+    start = text.find(head)
+    if start < 0:
+        return ""
+    start += len(head)
+    end = text.find(stop, start)
+    return text[start:end if end >= 0 else None].strip()
+
+
+def load_cases(roots, n, seed=20261009, max_marks=120, min_rows=0):
+    from ..decider_scope import commits
+    from ..marks import element_marks
+    cases = []
+    for root in roots:
+        for prompts in sorted(Path(root).glob("*/*/prompts.jsonl")):
+            for line in prompts.read_text().splitlines():
+                try:
+                    turn = json.loads(line)
+                except ValueError:
+                    continue
+                out = turn.get("out") or {}
+                acts = out.get("actions") or []
+                if not acts or acts[0].get("operation") != "TAP" or not acts[0].get("target"):
+                    continue
+                image = turn.get("image")
+                if not image or not (prompts.parent / image).exists():
+                    continue
+                rows = parse_rows(turn.get("text") or "")
+                if len(rows) < min_rows:
+                    continue
+                by_alias = dict(rows)
+                chosen = by_alias.get(acts[0]["target"])
+                if chosen is None or commits(chosen):
+                    continue
+                marks = element_marks([e for _, e in rows], limit=max_marks)
+                if not any(m.element is chosen for m in marks):
+                    continue
+                cases.append({"run": str(prompts.parent), "step": turn.get("step"), "text": turn["text"],
+                              "image": str(prompts.parent / image), "thought": out.get("thought") or "",
+                              "marks": marks, "want": next(m.id for m in marks if m.element is chosen),
+                              "want_label": chosen.label, "n_rows": len(rows)})
+    random.Random(seed).shuffle(cases)
+    return cases[:n]
+
+
+def question(case, framing):
+    from ..decider_scope import TARGET, admit
+    from ..decision_provider import Question
+    from ..marks import options
+    admit(TARGET, op="TAP")
+    request = section(case["text"], "Request: ")
+    marks = [m for m in case["marks"]]
+    if framing == "ground":
+        context = (f"An iPhone screen. The agent's next step, in its own words: {case['thought']}\n"
+                   "Pick the marked element that this step taps.")
+        instructions = "Which marked element does the step tap? Answer none if no mark fits."
+    else:
+        plan = section(case["text"], "Plan so far: ")
+        recent = section(case["text"], "Recent actions (oldest first):\n", "\n\n")
+        context = (f"An iPhone agent is working on this request: {request}\nIts plan so far: {plan or '(none)'}\n"
+                   f"Recent actions:\n{recent or '(none)'}\nPick the element the agent should tap next.")
+        instructions = ("Which marked element should be tapped next to progress the request? Answer none if the "
+                        "next step is not a tap on one of these elements.")
+    context += "\nEach option is a marked element: id, role, label, @x,y,w,h in % of the screen."
+    return context, {"target": Question("choice", instructions, options(marks))}
+
+
+def hedged(provider, context, questions, image, copies):
+    """The first of ``copies`` identical requests sent at once (each is billed)."""
+    if copies <= 1:
+        return provider.ask(context, questions, image=image, timeout=30)
+    pool = concurrent.futures.ThreadPoolExecutor(copies)
+    futures = [pool.submit(provider.ask, context, questions, image, 30) for _ in range(copies)]
+    try:
+        for future in concurrent.futures.as_completed(futures):
+            if future.exception() is None:
+                return future.result()
+        return futures[0].result()
+    finally:
+        pool.shutdown(wait=False)
+
+
+def ask(provider, case, framing, with_image, copies=1):
+    from ..marks import draw
+    context, questions = question(case, framing)
+    image = draw(Path(case["image"]).read_bytes(), case["marks"]) if with_image else None
+    started = time.monotonic()
+    try:
+        answers = hedged(provider, context, questions, image, copies)
+    except Exception as error:
+        return {"error": f"{type(error).__name__}: {str(error)[:200]}", "s": time.monotonic() - started}
+    answer = answers.get("target")
+    if answer is None:
+        return {"error": "malformed", "s": time.monotonic() - started}
+    return {"choice": answer.choice, "top": answer.top, "s": time.monotonic() - started}
+
+
+def summarize(results, provider):
+    done = [r for r in results if "choice" in r]
+    agree = [r for r in done if r["choice"] == r["want"]]
+    lat = sorted(r["s"] for r in results)
+    calib = []
+    lo = 0.0
+    for hi in BUCKETS:
+        bucket = [r for r in done if lo <= (r["top"] or 0) < hi]
+        if bucket:
+            calib.append({"range": [lo, min(hi, 1.0)], "n": len(bucket),
+                          "agree": round(sum(r["choice"] == r["want"] for r in bucket) / len(bucket), 3),
+                          "mean_top": round(statistics.mean(r["top"] or 0 for r in bucket), 3)})
+        lo = hi
+    confident = [r for r in done if (r["top"] or 0) >= .9 and r["choice"] != "none"]
+    return {"cases": len(results), "answered": len(done), "errors": len(results) - len(done),
+            "agreement": round(len(agree) / len(results), 3) if results else None,
+            "abstained": sum(r["choice"] == "none" for r in done),
+            "confident_n": len(confident),
+            "confident_agreement": round(sum(r["choice"] == r["want"] for r in confident) / len(confident), 3)
+            if confident else None,
+            "p50_s": round(lat[len(lat) // 2], 3) if lat else None,
+            "p90_s": round(lat[int(len(lat) * .9) - 1], 3) if lat else None,
+            "usd": round(provider.cost_usd, 4), "input_tokens": provider.input_tokens, "calibration": calib,
+            "error_kinds": sorted({r["error"][:80] for r in results if "error" in r})[:5]}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.decide_gate")
+    parser.add_argument("--runs", action="append", required=True)
+    parser.add_argument("--n", type=int, default=150)
+    parser.add_argument("--deciders", default="jev,openai-text,openai-marks")
+    parser.add_argument("--framings", default="ground,delegate")
+    parser.add_argument("--max-marks", type=int, default=120)
+    parser.add_argument("--workers", type=int, default=4)
+    parser.add_argument("--min-rows", type=int, default=0, help="only screens with at least this many rows")
+    parser.add_argument("--hedge", type=int, default=1, help="identical requests sent at once; the first answer wins")
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args(argv)
+    from ..decision_provider import JevDecisions, OpenAIDecisions
+    cases = load_cases(args.runs, args.n, max_marks=args.max_marks, min_rows=args.min_rows)
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    report = {"cases": len(cases), "max_marks": args.max_marks, "hedge": args.hedge, "min_rows": args.min_rows,
+              "marks_per_case_p50": statistics.median(len(c["marks"]) for c in cases) if cases else None,
+              "results": {}}
+    for decider in args.deciders.split(","):
+        for framing in args.framings.split(","):
+            provider = JevDecisions() if decider == "jev" else OpenAIDecisions()
+            with_image = decider == "openai-marks"
+            with concurrent.futures.ThreadPoolExecutor(args.workers) as pool:
+                rows = list(pool.map(lambda c: {**ask(provider, c, framing, with_image, args.hedge), "want": c["want"],
+                                                "marks": len(c["marks"]),
+                                                "run": c["run"], "step": c["step"]}, cases))
+            key = f"{decider}/{framing}"
+            report["results"][key] = summarize(rows, provider)
+            (out / f"{decider}-{framing}.jsonl").write_text("\n".join(json.dumps(r) for r in rows) + "\n")
+            print(key, json.dumps({k: v for k, v in report["results"][key].items() if k != "calibration"}))
+    (out / "report.json").write_text(json.dumps(report, indent=1))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())