mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/evals/app_survey.py

@@ -0,0 +1,149 @@
+"""How much of each app's screen the accessibility tree actually exposes.
+
+An accessibility-first agent is only as good as the tree an app publishes, and
+that depends on how the app is built: UIKit/SwiftUI publish nearly everything,
+React Native maps to native views, Flutter publishes a separate semantics tree,
+web views go through WebKit, and games or canvases often publish nothing.
+
+For each app this launches it (no taps, typing or scrolling), reads the tree the
+agent would read, takes one screenshot, recognizes its text with on-device
+Apple Vision, and reports how much of the visible text the tree also carries.
+
+Privacy: everything runs locally. Screenshots and recognized text stay in
+memory; no model is called; the report holds counts and ratios only.
+
+    python -m mobile_agent.evals.app_survey --wda-url http://127.0.0.1:8100 \
+        --bundles com.apple.Preferences com.spotify.client --out survey.json
+"""
+
+import argparse
+import base64
+import json
+import re
+import subprocess
+import time
+import urllib.request
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from ..compose import build_target_driver
+from ..paths import build_dir
+from ..wda.observe import WDA_SOURCE_PATH
+
+OCR = build_dir() / "ocr"
+ACTIONABLE = {"Button", "Cell", "Link", "TextField", "SearchField", "TextView", "Switch", "Slider",
+              "Tab", "Key", "SegmentedControl", "PickerWheel", "Stepper", "Toggle"}
+# OCR lines worth scoring: confident and long enough to be real text.
+MIN_CONFIDENCE, MIN_CHARS = .5, 3
+SETTLE_SECONDS = 3.0
+
+
+def normalize(text):
+    return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text.casefold())).strip()
+
+
+def text_recall(lines, corpus):
+    """Share of visible text lines the tree also carries (substring or 80% of words)."""
+    corpus_text = normalize(" ".join(corpus))
+    words = set(corpus_text.split())
+    scored = covered = 0
+    for line in lines:
+        norm = normalize(line)
+        if len(norm) < MIN_CHARS:
+            continue
+        scored += 1
+        tokens = norm.split()
+        if norm in corpus_text or (tokens and sum(t in words for t in tokens) / len(tokens) >= .8):
+            covered += 1
+    return (covered / scored if scored else None), scored
+
+
+def raw_metrics(xml):
+    """Counts from the raw tree, including what the agent's parser drops."""
+    root = ET.fromstring(xml)
+    nodes = list(root.iter())
+    roles = [node.tag.removeprefix("XCUIElementType") for node in nodes]
+    unlabeled = sum(1 for node, role in zip(nodes, roles)
+                    if role in ACTIONABLE and not (node.attrib.get("label") or node.attrib.get("value")))
+    return {"nodes": len(nodes), "webviews": roles.count("WebView"),
+            "actionable_raw": sum(role in ACTIONABLE for role in roles), "unlabeled_actionable": unlabeled}
+
+
+# The status bar (clock, signal, battery) belongs to SpringBoard, not the app.
+STATUS_BAR_BAND = .06
+
+
+def ocr_lines(png):
+    completed = subprocess.run([str(OCR)], input=png, capture_output=True, timeout=30)
+    if completed.returncode:
+        return []
+    return [row["text"] for row in json.loads(completed.stdout)
+            if row.get("confidence", 0) >= MIN_CONFIDENCE and isinstance(row.get("text"), str)
+            and row.get("rect", [0, 1])[1] >= STATUS_BAR_BAND]
+
+
+def classify(metrics):
+    recall, elements, lines = metrics.get("text_recall"), metrics.get("elements", 0), metrics.get("ocr_lines", 0)
+    if metrics.get("foreground") != metrics["bundle_id"]:
+        return "not_foreground"
+    if elements <= 3 and lines >= 5:
+        return "opaque"          # drawn surface: games, canvases, video
+    if recall is None:
+        return "no_text"
+    if recall >= .85:
+        return "good"
+    if recall >= .6:
+        return "partial"
+    return "poor"
+
+
+def survey_app(driver, bundle_id):
+    metrics = {"bundle_id": bundle_id}
+    try:
+        driver.call("POST", "/wda/apps/activate", {"bundleId": bundle_id}, timeout=20)
+        time.sleep(SETTLE_SECONDS)
+        driver._stable = None
+        started = time.monotonic()
+        snapshot = driver.observe_ready(timeout=20)
+        metrics["read_ms"] = round((time.monotonic() - started) * 1000)
+        metrics["foreground"] = snapshot.bundle_id
+        xml = driver.call("GET", WDA_SOURCE_PATH, timeout=30)
+        metrics.update(raw_metrics(xml))
+        corpus = [f"{e.label} {e.value}" for e in snapshot.elements] + snapshot.text.split("\n")
+        metrics["elements"] = len(snapshot.elements)
+        metrics["actionable"] = sum(e.role in ACTIONABLE for e in snapshot.elements)
+        image = driver.capture_preview(timeout=10)
+        lines = ocr_lines(base64.b64decode(image.split(",", 1)[1]))
+        metrics["ocr_lines"] = len(lines)
+        metrics["text_recall"], metrics["scored_lines"] = text_recall(lines, corpus)
+    except Exception as error:
+        metrics["error"] = type(error).__name__
+    metrics["class"] = classify(metrics) if "error" not in metrics else "error"
+    return metrics
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Accessibility coverage across installed apps (local only)")
+    parser.add_argument("--wda-url", default="http://127.0.0.1:8100")
+    parser.add_argument("--bundles", nargs="+", required=True)
+    parser.add_argument("--out", help="Write per-app metrics (numbers only) as JSON")
+    args = parser.parse_args()
+    with urllib.request.urlopen(args.wda_url + "/status") as response:
+        session = json.load(response)["sessionId"]
+    driver = build_target_driver(wda_url=args.wda_url, session=session)
+    rows = []
+    for bundle in args.bundles:
+        row = survey_app(driver, bundle)
+        rows.append(row)
+        recall = row.get("text_recall")
+        print(f"{bundle:45} {row['class']:14} elements={row.get('elements', '-'):>4} "
+              f"recall={'-' if recall is None else f'{recall:.2f}':>5} lines={row.get('ocr_lines', '-'):>3} "
+              f"unlabeled={row.get('unlabeled_actionable', '-'):>3} webviews={row.get('webviews', '-')} "
+              f"read={row.get('read_ms', '-')}ms", flush=True)
+    driver.call("POST", "/wda/pressButton", {"name": "home"}, timeout=10)
+    if args.out:
+        Path(args.out).write_text(json.dumps(rows, indent=1))
+
+
+if __name__ == "__main__":
+    main()

mobile_agent/evals/development/__init__.py

@@ -0,0 +1 @@
+"""Offline scoring of pre-registered development evaluations."""

mobile_agent/evals/development/score.py

@@ -0,0 +1,276 @@
+"""Score paired development runs from independent, pre-registered evidence.
+
+This is an offline reader. It never builds an app, runs an agent, or probes a device.
+The harness must write oracle.json without consulting claims.json.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections import Counter
+from pathlib import Path
+
+
+class InvalidRun(ValueError):
+    """The input cannot be joined to its registered run."""
+
+
+def _read(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as error:
+        raise InvalidRun(f"cannot read {path}: {error}") from error
+
+
+def _digest(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _sha(value):
+    return isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+
+
+def _artifact_matches(run_dir: Path, ref, digest):
+    if not isinstance(ref, str) or not ref or not _sha(digest):
+        return False
+    try:
+        path = (run_dir / ref).resolve()
+    except (OSError, RuntimeError):
+        return False
+    if not path.is_relative_to(run_dir.resolve()) or not path.is_file():
+        return False
+    try:
+        if path.stat().st_size > 5_000_000 or _digest(path.read_bytes()) != digest:
+            return False
+    except OSError:
+        return False
+    return True
+
+
+def _evidence(run_dir: Path, check):
+    """Missing/tampered evidence is unknown; a claimed boolean is insufficient."""
+    if not _artifact_matches(run_dir, check.get("evidence"), check.get("sha256")):
+        return None
+    return check.get("ok") if type(check.get("ok")) is bool else None
+
+
+def _manifest(path: Path):
+    try:
+        raw = path.read_bytes()
+        manifest = json.loads(raw)
+    except (OSError, UnicodeError, json.JSONDecodeError) as error:
+        raise InvalidRun(f"cannot read manifest {path}: {error}") from error
+    if not isinstance(manifest, dict) or type(manifest.get("schema")) is not int or manifest["schema"] != 1:
+        raise InvalidRun("unsupported manifest schema")
+    cases = manifest.get("cases")
+    if not isinstance(cases, list) or not cases:
+        raise InvalidRun("manifest needs registered cases")
+    by_id = {}
+    for case in cases:
+        if not isinstance(case, dict) or not isinstance(case.get("id"), str) or not case["id"]:
+            raise InvalidRun("invalid case id")
+        if case["id"] in by_id:
+            raise InvalidRun(f"duplicate case {case['id']}")
+        if case.get("family") not in {"automation", "testing", "ux", "reliability"}:
+            raise InvalidRun(f"invalid family for {case['id']}")
+        if not _sha(case.get("checkSha256")):
+            raise InvalidRun(f"missing registered check digest for {case['id']}")
+        if type(case.get("maxAttempts")) is not int or not 1 <= case["maxAttempts"] <= 10:
+            raise InvalidRun(f"invalid attempt limit for {case['id']}")
+        groups = [case.get("regressionChecks"), case.get("criticalChecks")]
+        if not isinstance(case.get("bugCheck"), str) or any(not isinstance(g, list) or not g for g in groups):
+            raise InvalidRun(f"invalid checks for {case['id']}")
+        names = [case["bugCheck"], *groups[0], *groups[1]]
+        if any(not isinstance(name, str) or not name for name in names) or len(set(names)) != len(names):
+            raise InvalidRun(f"duplicate or empty check for {case['id']}")
+        by_id[case["id"]] = case
+    return _digest(raw), by_id
+
+
+def _header(document, manifest_sha):
+    if not isinstance(document, dict) or type(document.get("schema")) is not int or document["schema"] != 1:
+        raise InvalidRun("unsupported run schema")
+    if document.get("manifestSha256") != manifest_sha:
+        raise InvalidRun("run does not match manifest bytes")
+    if not isinstance(document.get("runId"), str) or not document["runId"] or not _sha(document.get("configSha256")):
+        raise InvalidRun("missing run or configuration identity")
+    if not isinstance(document.get("attempts"), list):
+        raise InvalidRun("attempts must be a list")
+
+
+def _attempts(document, cases):
+    rows = {}
+    for row in document["attempts"]:
+        if not isinstance(row, dict):
+            raise InvalidRun("invalid attempt")
+        case_id, ordinal = row.get("caseId"), row.get("attempt")
+        if (not isinstance(case_id, str) or case_id not in cases or type(ordinal) is not int
+                or ordinal < 1 or ordinal > cases[case_id]["maxAttempts"]):
+            raise InvalidRun(f"unknown case or attempt: {case_id}/{ordinal}")
+        key = (case_id, ordinal)
+        if key in rows:
+            raise InvalidRun(f"duplicate attempt {case_id}/{ordinal}")
+        rows[key] = row
+    return rows
+
+
+def _checks(run_dir, observation, case):
+    expected = {case["bugCheck"], *case["regressionChecks"], *case["criticalChecks"]}
+    entries = observation.get("checks")
+    if not isinstance(entries, list):
+        return {name: None for name in expected}
+    names = [entry.get("id") for entry in entries if isinstance(entry, dict)]
+    if (len(names) != len(entries) or any(not isinstance(name, str) for name in names)
+            or len(set(names)) != len(names) or not set(names) <= expected):
+        raise InvalidRun(f"check IDs differ from manifest for {case['id']}")
+    observed = {entry["id"]: _evidence(run_dir, entry) for entry in entries}
+    return {name: observed.get(name) for name in expected}
+
+
+def score(manifest_path: Path, run_dir: Path):
+    """Return a JSON-serializable census; no missing case silently disappears."""
+    manifest_sha, cases = _manifest(manifest_path)
+    claims = _read(run_dir / "claims.json")
+    oracle = _read(run_dir / "oracle.json")
+    for document in (claims, oracle):
+        _header(document, manifest_sha)
+    if (claims["runId"], claims["configSha256"]) != (oracle["runId"], oracle["configSha256"]):
+        raise InvalidRun("claims and oracle belong to different runs")
+    builds = (oracle.get("baselineBuildSha256"), oracle.get("fixedBuildSha256"))
+    if not all(map(_sha, builds)) or builds[0] == builds[1]:
+        raise InvalidRun("distinct baseline and fixed build digests required")
+    claim_rows = _attempts(claims, cases)
+    oracle_rows = _attempts(oracle, cases)
+    for case_id in cases:
+        ordinals = sorted(n for cid, n in claim_rows.keys() | oracle_rows.keys() if cid == case_id)
+        if ordinals != list(range(1, len(ordinals) + 1)):
+            raise InvalidRun(f"non-contiguous attempts for {case_id}")
+
+    results = []
+    for case_id, case in cases.items():
+        attempts = []
+        critical_veto = False
+        verification_veto = False
+        for ordinal in range(1, case["maxAttempts"] + 1):
+            key = (case_id, ordinal)
+            if key not in claim_rows and key not in oracle_rows:
+                continue
+            claim = claim_rows.get(key)
+            evidence = oracle_rows.get(key)
+            baseline, fixed = {}, {}
+            comparable = False
+            if claim is not None and (claim.get("baselineVerdict") not in {"bug", "clean", "unknown"}
+                                      or claim.get("fixedVerdict") not in {"fixed", "broken", "unknown"}):
+                raise InvalidRun(f"invalid agent verdict for {case_id}/{ordinal}")
+            if evidence is not None:
+                arms = [evidence.get("baseline"), evidence.get("fixed")]
+                if not all(isinstance(arm, dict) for arm in arms):
+                    raise InvalidRun(f"missing paired arms for {case_id}/{ordinal}")
+                if any(arm.get("buildSha256") != build for arm, build in zip(arms, builds)):
+                    raise InvalidRun(f"build mismatch for {case_id}/{ordinal}")
+                seeds = [arm.get("startStateSha256") for arm in arms]
+                assertion = [arm.get("assertionSha256") for arm in arms]
+                setup = [arm.get("setupSha256") for arm in arms]
+                comparable = (seeds[0] == seeds[1] and _sha(seeds[0])
+                              and assertion == [case["checkSha256"]] * 2
+                              and setup[0] == setup[1] and _sha(setup[0]))
+                for arm, target in zip(arms, (baseline, fixed)):
+                    if arm.get("reached") is not None and type(arm.get("reached")) is not bool:
+                        raise InvalidRun(f"invalid reach for {case_id}/{ordinal}")
+                    target["reached"] = arm.get("reached")
+                    if arm.get("reached") is True:
+                        target.update(_checks(run_dir, arm, case))
+                    else:
+                        target.update({name: None for name in [case["bugCheck"], *case["regressionChecks"],
+                                                               *case["criticalChecks"]]})
+            if any(baseline.get(name) is not True or fixed.get(name) is not True
+                   for name in case["criticalChecks"]):
+                critical_veto = True
+            if (not comparable or baseline.get("reached") is not True or fixed.get("reached") is not True
+                    or baseline.get(case["bugCheck"]) is not False or fixed.get(case["bugCheck"]) is not True
+                    or any(baseline.get(name) is not True or fixed.get(name) is not True
+                           for name in case["regressionChecks"])):
+                verification_veto = True
+            if claim is None or evidence is None or not comparable:
+                category = "inconclusive"
+            elif baseline["reached"] is False:
+                category = "unreached"
+            elif (baseline["reached"] is not True or baseline[case["bugCheck"]] is None
+                  or baseline[case["bugCheck"]] is True):
+                category = "inconclusive"
+            elif claim["baselineVerdict"] == "bug":
+                category = "detected"
+            else:
+                category = "survived"
+            fixed_clean = (comparable and fixed.get("reached") is True
+                           and fixed.get(case["bugCheck"]) is True
+                           and all(fixed.get(name) is True for name in
+                                   case["regressionChecks"] + case["criticalChecks"]))
+            fixed_proven = (category == "detected" and fixed_clean
+                            and all(baseline.get(name) is True for name in
+                                    case["regressionChecks"] + case["criticalChecks"]))
+            fixed_failed = (comparable and fixed.get("reached") is True and
+                            any(fixed.get(name) is False for name in
+                                [case["bugCheck"], *case["regressionChecks"], *case["criticalChecks"]]))
+            repro = claim.get("repro") if claim else None
+            repro_proven = (isinstance(repro, dict) and evidence is not None
+                            and _artifact_matches(run_dir, repro.get("path"), repro.get("sha256"))
+                            and evidence.get("replayedReproSha256") == repro.get("sha256")
+                            and comparable and baseline.get("reached") is True and fixed.get("reached") is True
+                            and baseline.get(case["bugCheck"]) is False and fixed.get(case["bugCheck"]) is True)
+            baseline_false = bool(claim and claim["baselineVerdict"] == "clean" and comparable
+                                  and baseline.get(case["bugCheck"]) is False)
+            fixed_false = bool(claim and claim["fixedVerdict"] == "fixed" and fixed_failed)
+            baseline_unsupported = bool(claim and claim["baselineVerdict"] == "clean"
+                                        and (not comparable or baseline.get(case["bugCheck"]) is None))
+            fixed_unsupported = bool(claim and claim["fixedVerdict"] == "fixed"
+                                     and not fixed_proven and not fixed_failed)
+            attempts.append({
+                "attempt": ordinal,
+                "category": category,
+                "reproProven": bool(repro_proven),
+                "falseSuccess": int(baseline_false) + int(fixed_false),
+                "unsupportedSuccess": int(baseline_unsupported) + int(fixed_unsupported),
+                "fixedArmFalsePositive": bool(claim and claim["fixedVerdict"] == "broken" and fixed_clean),
+                "fixProven": bool(fixed_proven and claim["fixedVerdict"] == "fixed"),
+            })
+        first = attempts[0]["category"] if attempts else "unreached"
+        eventual = "detected" if any(row["category"] == "detected" for row in attempts) else (
+            "survived" if any(row["category"] == "survived" for row in attempts) else (
+                "inconclusive" if any(row["category"] == "inconclusive" for row in attempts) else "unreached"))
+        results.append({"caseId": case_id, "family": case["family"], "firstAttempt": first,
+                        "eventual": eventual, "recovered": first != "detected" and eventual == "detected",
+                        "reproProven": not verification_veto and any(row["reproProven"] for row in attempts),
+                        "verifiedFix": not critical_veto and not verification_veto
+                        and any(row["fixProven"] for row in attempts),
+                        "criticalEvidenceVeto": critical_veto, "verificationEvidenceVeto": verification_veto,
+                        "attempts": attempts})
+    by_family = {}
+    for family in ("automation", "testing", "ux", "reliability"):
+        group = [row for row in results if row["family"] == family]
+        by_family[family] = {"registered": len(group),
+                             "firstAttempt": dict(Counter(row["firstAttempt"] for row in group)),
+                             "eventual": dict(Counter(row["eventual"] for row in group)),
+                             "recovered": sum(row["recovered"] for row in group),
+                             "reproProven": sum(row["reproProven"] for row in group),
+                             "verifiedFix": sum(row["verifiedFix"] for row in group)}
+    return {"runId": oracle["runId"], "manifestSha256": manifest_sha,
+            "configSha256": oracle["configSha256"], "registered": len(cases),
+            "families": by_family, "falseSuccess": sum(a["falseSuccess"] for r in results for a in r["attempts"]),
+            "unsupportedSuccess": sum(a["unsupportedSuccess"] for r in results for a in r["attempts"]),
+            "fixedArmFalsePositive": sum(a["fixedArmFalsePositive"] for r in results for a in r["attempts"]),
+            "criticalEvidenceVeto": sum(r["criticalEvidenceVeto"] for r in results), "cases": results}
+
+
+def main():
+    parser = argparse.ArgumentParser(description="score an offline paired development evaluation")
+    parser.add_argument("run_dir", type=Path)
+    parser.add_argument("--manifest", type=Path, required=True)
+    args = parser.parse_args()
+    print(json.dumps(score(args.manifest, args.run_dir), indent=2))
+
+
+if __name__ == "__main__":
+    main()