mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/bench/evals/checkpoint.py

@@ -0,0 +1,214 @@
+"""L2 prototype: checkpointed starts, by snapshotting the iOSWorld apps' data on a simulator.
+
+    python -m mobile_agent.bench.evals.checkpoint save    --udid UDID --repo IOSWORLD --name NAME --root DIR
+    python -m mobile_agent.bench.evals.checkpoint restore --udid UDID --repo IOSWORLD --name NAME --root DIR
+    python -m mobile_agent.bench.evals.checkpoint digest  --udid UDID --repo IOSWORLD
+
+A checkpoint is what iOSWorld's own reset wipes. For each app, that's the data folders of its data container:
+UserDefaults, Application Support, caches, cookies, saved state, WebKit, Documents. It also includes the shared
+app-group containers the suite apps use. Saving and restoring copy with APFS clones (``cp -c``), so a checkpoint
+takes about a second and almost no disk until an app writes.
+
+Two uses:
+- A reset: restoring the freshly seeded checkpoint replaces iOSWorld's wipe-and-relaunch-every-app reset.
+- A crux start: a checkpoint taken just before a task's hard part lets a run begin there.
+
+``digest`` hashes the persistent folders (Preferences, Application Support, Documents), so a restore can be
+checked against its save. It refuses a simulator whose name doesn't start with one of ``--allow`` (default "Eval
+Lab"), so the tuning loop's simulators are never touched.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import pathlib
+import plistlib
+import shutil
+import subprocess
+import time
+
+DATA_DIRS = ("Library/Preferences", "Library/Application Support", "Library/Caches", "Library/Cookies",
+             "Library/SplashBoard", "Library/Saved Application State", "Library/WebKit", "Documents")
+PERSISTENT = ("Library/Preferences", "Library/Application Support", "Documents")
+SHARED_GROUPS = ("group.com.iosworld.benchmark.workspacesuite",)
+DEVICES = pathlib.Path("~/Library/Developer/CoreSimulator/Devices").expanduser()
+
+
+def device_name(udid):
+    out = (subprocess.run(["xcrun", "simctl", "list", "devices", "-j"], capture_output=True, text=True,
+                          check=True).stdout)
+    for devices in json.loads(out)["devices"].values():
+        for d in devices:
+            if d["udid"] == udid:
+                return d["name"]
+    return None
+
+
+def guard(udid, allow=("Eval Lab",)):
+    name = device_name(udid)
+    if not name or not name.startswith(tuple(allow)):
+        raise SystemExit(f"refusing simulator {udid} ({name!r}): only simulators named {allow} are touched")
+    return name
+
+
+def manifest(repo):
+    """{app: bundle id} from iOSWorld's bootstrap manifest."""
+    path = next(iter(sorted(pathlib.Path(repo).expanduser().rglob(".app_manifest.json"))), None)
+    if path is None:
+        raise FileNotFoundError("no .app_manifest.json: run iOSWorld's bootstrap first")
+    return {app: entry["bundle_id"] for app, entry in json.loads(path.read_text()).items() if entry.get("bundle_id")}
+
+
+def app_container(udid, bundle):
+    out = subprocess.run(["xcrun", "simctl", "get_app_container", udid, bundle, "data"], capture_output=True, text=True)
+    path = out.stdout.strip()
+    return pathlib.Path(path) if out.returncode == 0 and path else None
+
+
+def group_containers(udid, groups=SHARED_GROUPS):
+    root = DEVICES / udid / "data" / "Containers" / "Shared" / "AppGroup"
+    out = {}
+    for folder in root.iterdir() if root.is_dir() else ():
+        meta = folder / ".com.apple.mobile_container_manager.metadata.plist"
+        try:
+            group = plistlib.loads(meta.read_bytes()).get("MCMMetadataIdentifier")
+        except (OSError, plistlib.InvalidFileException):
+            continue
+        if group in groups:
+            out[group] = folder
+    return out
+
+
+def terminate(udid, bundles):
+    for bundle in bundles:
+        subprocess.run(["xcrun", "simctl", "terminate", udid, bundle], capture_output=True)
+
+
+def _clear(folder):
+    for entry in folder.iterdir() if folder.is_dir() else ():
+        if entry.name.startswith(".com.apple.mobile_container_manager"):
+            continue
+        if entry.is_dir() and not entry.is_symlink():
+            shutil.rmtree(entry)
+        else:
+            entry.unlink()
+
+
+def _clone(src, dst):
+    """Copy ``src``'s contents into ``dst`` with APFS clones (``cp -c``; a plain copy elsewhere)."""
+    dst.mkdir(parents=True, exist_ok=True)
+    entries = [e for e in src.iterdir() if not e.name.startswith(".com.apple.mobile_container_manager")]
+    if entries:
+        subprocess.run(["cp", "-cRp", *map(str, entries), str(dst)], check=True)
+
+
+def save(udid, bundles, dest):
+    """Snapshot every app's data folders and the shared groups into ``dest`` (which must not exist)."""
+    dest = pathlib.Path(dest)
+    if dest.exists():
+        raise FileExistsError(f"{dest} exists: a checkpoint is never overwritten")
+    started = time.monotonic()
+    terminate(udid, bundles.values())
+    for app, bundle in bundles.items():
+        container = app_container(udid, bundle)
+        if container is None:
+            continue
+        for rel in DATA_DIRS:
+            if (container / rel).is_dir():
+                _clone(container / rel, dest / "apps" / bundle / rel)
+    for group, folder in group_containers(udid).items():
+        _clone(folder, dest / "groups" / group)
+    meta = {"udid": udid, "saved": time.strftime("%Y-%m-%dT%H:%M:%S"), "apps": bundles,
+            "digest": digest(udid, bundles), "seconds": round(time.monotonic() - started, 2)}
+    (dest / "checkpoint.json").write_text(json.dumps(meta, indent=1))
+    return meta
+
+
+def restore(udid, bundles, src):
+    """Put a checkpoint back: every app's data folders and the shared groups, as saved. Returns (seconds, ok)."""
+    src = pathlib.Path(src)
+    meta = json.loads((src / "checkpoint.json").read_text())
+    if not isinstance(meta, dict) or meta.get("apps") != bundles or not isinstance(meta.get("digest"), str):
+        raise ValueError("checkpoint metadata doesn't match the apps to restore; no app data was changed")
+    started = time.monotonic()
+    roots = [(f"{bundles[app]}/{rel}", src / "apps" / bundles[app] / rel)
+             for app in sorted(bundles) for rel in PERSISTENT]
+    groups = src / "groups"
+    roots += [(f"group/{folder.name}", folder) for folder in sorted(groups.iterdir())] if groups.is_dir() else []
+    if _digest(roots) != meta["digest"]:
+        return round(time.monotonic() - started, 2), False
+    terminate(udid, bundles.values())
+    for app, bundle in bundles.items():
+        container = app_container(udid, bundle)
+        if container is None:
+            continue
+        for rel in DATA_DIRS:
+            target = container / rel
+            if target.is_dir():
+                _clear(target)
+            saved = src / "apps" / bundle / rel
+            if saved.is_dir():
+                _clone(saved, target)
+    for group, folder in group_containers(udid).items():
+        _clear(folder)
+        if (src / "groups" / group).is_dir():
+            _clone(src / "groups" / group, folder)
+    seconds = time.monotonic() - started
+    return round(seconds, 2), digest(udid, bundles) == meta["digest"]
+
+
+def digest(udid, bundles):
+    """sha256 over the persistent folders' files (paths and bytes), apps in name order, then the shared groups."""
+    roots = []
+    for app in sorted(bundles):
+        container = app_container(udid, bundles[app])
+        if container is not None:
+            roots += [(f"{bundles[app]}/{rel}", container / rel) for rel in PERSISTENT]
+    roots += [(f"group/{g}", f) for g, f in sorted(group_containers(udid).items())]
+    return _digest(roots)
+
+
+def _digest(roots):
+    h = hashlib.sha256()
+    for label, root in roots:
+        if not root.is_dir():
+            continue
+        for dirpath, dirnames, filenames in os.walk(root):
+            dirnames.sort()
+            for name in sorted(filenames):
+                if name.startswith(".com.apple.mobile_container_manager"):
+                    continue
+                path = pathlib.Path(dirpath) / name
+                h.update(f"{label}/{path.relative_to(root)}".encode())
+                h.update(path.read_bytes())
+    return h.hexdigest()
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.evals.checkpoint",
+                                     description=__doc__.split("\n\n")[0])
+    parser.add_argument("action", choices=("save", "restore", "digest"))
+    parser.add_argument("--udid", required=True)
+    parser.add_argument("--repo", required=True, help="the iOSWorld checkout (or a folder "
+                        "holding its bootstrap manifest)")
+    parser.add_argument("--name")
+    parser.add_argument("--root", default="~/mobster-work/evals/checkpoints")
+    parser.add_argument("--allow", nargs="*", default=["Eval Lab"])
+    args = parser.parse_args(argv)
+    guard(args.udid, tuple(args.allow))
+    bundles = manifest(args.repo)
+    folder = pathlib.Path(args.root).expanduser() / (args.name or "")
+    if args.action == "save":
+        print(json.dumps({k: v for k, v in save(args.udid, bundles, folder).items() if k != "apps"}))
+    elif args.action == "restore":
+        seconds, ok = restore(args.udid, bundles, folder)
+        print(json.dumps({"restored": str(folder), "seconds": seconds, "digest_matches": ok}))
+        return 0 if ok else 1
+    else:
+        print(digest(args.udid, bundles))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mobile_agent/bench/evals/irt.py

@@ -0,0 +1,219 @@
+"""L2, offline: item response theory over every judged iOSWorld run, and the informative subset it implies.
+
+    python -m mobile_agent.bench.evals.irt --runs DIR... --pairs pairs.json [--k 10] --out DIR
+
+A respondent is one engine configuration (a batch's arm); an item is a task. The 2PL model says a respondent of
+ability theta passes task i with probability sigmoid(a_i (theta - b_i)): b_i is the task's difficulty, a_i how
+sharply it separates weaker from stronger engines. Fitted by maximum a posteriori with weak priors (theta ~ N(0, 1),
+b ~ N(0, 2^2), log a ~ N(0, 0.5^2)), plain gradient ascent, pure Python. Repeated runs of a task by one respondent
+count as binomial trials.
+
+A task always passed or always failed carries no signal about a delta. The informative subset is the k tasks with
+the most Fisher information, a^2 p (1 - p), at the ability of the engine being tuned.
+
+The backtest: for each paired experiment in ``pairs.json`` ({"name", "a": [respondent], "b": [respondent]}), refit
+without its runs, take the k most informative of its tasks, and compare the subset's delta with the full delta.
+The subset delta is computed two ways: the plain pass-rate difference on the subset, and an IRT estimate of the
+full set (fit each arm's ability on the subset alone, then predict every task). Random k-task subsets give the
+baseline error. With few experiments the backtest is a sanity check, not a validation.
+"""
+
+import argparse
+import collections
+import json
+import math
+import pathlib
+import random
+import statistics
+
+from . import traces as T
+
+
+def sigmoid(z):
+    return 1 / (1 + math.exp(-max(-30.0, min(30.0, z))))
+
+
+def responses(runs, clean_only=False):
+    """{(respondent, task): [passes, trials]} over judged runs. A respondent is "batch:arm"."""
+    out = collections.defaultdict(lambda: [0, 0])
+    for r in runs:
+        if not r.judged or (clean_only and not r.clean):
+            continue
+        cell = out[(f"{r.batch}:{r.arm}", r.task)]
+        cell[0] += bool(r.passed)
+        cell[1] += 1
+    return dict(out)
+
+
+def fit(cells, iterations=3000, rate=.05):
+    """MAP 2PL fit. ``cells``: {(respondent, task): (passes, trials)}. Returns {"theta", "a", "b", "loglik"}."""
+    people = sorted({p for p, _ in cells})
+    items = sorted({i for _, i in cells})
+    theta = {p: 0.0 for p in people}
+    loga = {i: 0.0 for i in items}
+    b = {i: 0.0 for i in items}
+    # Adam state
+    m = collections.defaultdict(float)
+    v = collections.defaultdict(float)
+    beta1, beta2, eps = .9, .999, 1e-8
+    data = [(p, i, s, n) for (p, i), (s, n) in cells.items()]
+    for step in range(1, iterations + 1):
+        g = collections.defaultdict(float)
+        for p, i, s, n in data:
+            a = math.exp(loga[i])
+            z = a * (theta[p] - b[i])
+            resid = s - n * sigmoid(z)
+            g[("t", p)] += resid * a
+            g[("b", i)] -= resid * a
+            g[("a", i)] += resid * z
+        for p in people:
+            g[("t", p)] -= theta[p]
+        for i in items:
+            g[("b", i)] -= b[i] / 4.0
+            g[("a", i)] -= loga[i] / .25
+        for key, grad in g.items():
+            m[key] = beta1 * m[key] + (1 - beta1) * grad
+            v[key] = beta2 * v[key] + (1 - beta2) * grad * grad
+            mh, vh = m[key] / (1 - beta1 ** step), v[key] / (1 - beta2 ** step)
+            delta = rate * mh / (math.sqrt(vh) + eps)
+            kind, name = key
+            if kind == "t":
+                theta[name] += delta
+            elif kind == "b":
+                b[name] += delta
+            else:
+                loga[name] += delta
+    ll = 0.0
+    for p, i, s, n in data:
+        q = sigmoid(math.exp(loga[i]) * (theta[p] - b[i]))
+        ll += s * math.log(max(q, 1e-12)) + (n - s) * math.log(max(1 - q, 1e-12))
+    return {"theta": theta, "a": {i: math.exp(loga[i]) for i in items}, "b": b, "loglik": ll}
+
+
+def ability(cells, params, iterations=400, rate=.05):
+    """The MAP ability of one respondent from its cells {task: (passes, trials)}, items held fixed."""
+    theta = 0.0
+    for _ in range(iterations):
+        grad = -theta
+        for task, (s, n) in cells.items():
+            if task in params["a"]:
+                a = params["a"][task]
+                grad += (s - n * sigmoid(a * (theta - params["b"][task]))) * a
+        theta += rate * grad
+    return theta
+
+
+def information(params, task, theta):
+    a = params["a"][task]
+    p = sigmoid(a * (theta - params["b"][task]))
+    return a * a * p * (1 - p)
+
+
+def informative(params, tasks, theta, k):
+    """The k tasks with the most Fisher information at ``theta``."""
+    tasks = [t for t in tasks if t in params["a"]]
+    return sorted(tasks, key=lambda t: -information(params, t, theta))[:k]
+
+
+def _task_mean(cells, respondents, tasks):
+    """Mean over tasks of each task's pass rate (tasks weigh equally, as the scoreboard's paired delta does)."""
+    rates = []
+    for t in tasks:
+        s = sum(cells.get((p, t), (0, 0))[0] for p in respondents)
+        n = sum(cells.get((p, t), (0, 0))[1] for p in respondents)
+        if n:
+            rates.append(s / n)
+    return statistics.mean(rates) if rates else None
+
+
+def backtest(cells, pairs, k=10, draws=200, seed=20261010, iterations=1500, fraction=None):
+    """One row per paired experiment: full delta, subset deltas (plain and IRT), and the random-subset baseline."""
+    rng = random.Random(seed)
+    rows = []
+    fixed_k = k
+    for pair in pairs:
+        a_resp, b_resp = pair["a"], pair["b"]
+        mine = set(a_resp) | set(b_resp)
+        shared = sorted({t for p, t in cells if p in a_resp} & {t for p, t in cells if p in b_resp})
+        k = max(3, round(fraction * len(shared))) if fraction else fixed_k
+        if len(shared) < k + 2:
+            continue
+        rest = {key: v for key, v in cells.items() if key[0] not in mine}
+        params = fit(rest, iterations=iterations)
+        known = [t for t in shared if t in params["a"]]
+        if len(known) < k:
+            continue
+        base_theta = statistics.mean([params["theta"][p] for p in params["theta"]] or [0.0])
+        anchor = pair.get("anchor")  # a respondent to centre the information on (the engine being tuned)
+        if anchor and anchor in params["theta"]:
+            base_theta = params["theta"][anchor]
+        subset = informative(params, known, base_theta, k)
+        full = _task_mean(cells, b_resp, shared) - _task_mean(cells, a_resp, shared)
+        plain = _task_mean(cells, b_resp, subset) - _task_mean(cells, a_resp, subset)
+
+        def irt_full(respondents, tasks):
+            agg = collections.defaultdict(lambda: [0, 0])
+            for p in respondents:
+                for t in tasks:
+                    s, n = cells.get((p, t), (0, 0))
+                    agg[t][0] += s
+                    agg[t][1] += n
+            th = ability({t: tuple(v) for t, v in agg.items() if v[1]}, params)
+            return statistics.mean(sigmoid(params["a"][t] * (th - params["b"][t])) for t in known)
+        irt_delta = irt_full(b_resp, subset) - irt_full(a_resp, subset)
+        rand = []
+        for _ in range(draws):
+            pick = rng.sample(shared, k)
+            rand.append(_task_mean(cells, b_resp, pick) - _task_mean(cells, a_resp, pick))
+        rows.append({"name": pair["name"], "tasks": len(shared), "k": k, "full_delta": round(full, 3),
+                     "subset": subset, "subset_delta": round(plain, 3), "irt_delta": round(irt_delta, 3),
+                     "subset_error": round(abs(plain - full), 3), "irt_error": round(abs(irt_delta - full), 3),
+                     "random_error_mean": round(statistics.mean(abs(x - full) for x in rand), 3),
+                     "random_error_p50": round(statistics.median(abs(x - full) for x in rand), 3),
+                     "sign_agrees": (plain > 0) == (full > 0) if plain and full else None,
+                     "random_sign_agrees": round(sum((x > 0) == (full > 0) for x in rand if x and full) / draws, 3)})
+    return rows
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.bench.evals.irt",
+                                     description=__doc__.split("\n\n")[0])
+    parser.add_argument("--runs", nargs="+", required=True)
+    parser.add_argument("--pairs", help="JSON list of paired experiments for the backtest")
+    parser.add_argument("--anchor", help="respondent whose ability centres the informative subset (batch:arm)")
+    parser.add_argument("--tasks", help="comma-separated tasks to pick the subset from (default: every task fitted)")
+    parser.add_argument("--k", type=int, default=10)
+    parser.add_argument("--fraction", type=float, help="backtest subsets as this share of each experiment's tasks")
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args(argv)
+    runs = T.discover(args.runs, suites=("iosworld",))
+    cells = responses(runs)
+    params = fit(cells)
+    theta = params["theta"].get(args.anchor, statistics.mean(params["theta"].values()))
+    pool = args.tasks.split(",") if args.tasks else sorted(params["a"])
+    passes = collections.defaultdict(lambda: [0, 0])
+    for (p, t), (s, n) in cells.items():
+        passes[t][0] += s
+        passes[t][1] += n
+    items = sorted(({"task": t, "a": round(params["a"][t], 3), "b": round(params["b"][t], 3),
+                     "info_at_anchor": round(information(params, t, theta), 4),
+                     "passes": passes[t][0], "trials": passes[t][1]} for t in params["a"]),
+                   key=lambda r: -r["info_at_anchor"])
+    report = {"respondents": len(params["theta"]), "tasks": len(params["a"]), "cells": len(cells),
+              "loglik": round(params["loglik"], 2), "anchor": args.anchor, "anchor_theta": round(theta, 3),
+              "subset": informative(params, pool, theta, args.k), "items": items,
+              "abilities": dict(sorted(((p, round(t, 3)) for p, t in params["theta"].items()), key=lambda kv: -kv[1]))}
+    if args.pairs:
+        report["backtest"] = backtest(cells, json.loads(pathlib.Path(args.pairs).read_text()), k=args.k,
+                                      fraction=args.fraction)
+    out = pathlib.Path(args.out).expanduser()
+    out.mkdir(parents=True, exist_ok=True)
+    (out / "irt.json").write_text(json.dumps(report, indent=1))
+    print(json.dumps({k: report[k] for k in ("respondents", "tasks", "anchor_theta", "subset")}))
+    for row in report.get("backtest") or ():
+        print(json.dumps({k: v for k, v in row.items() if k != "subset"}))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())