mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/contract_view.py

@@ -0,0 +1,84 @@
+"""What the picture shows: which rows a screen's bars hide, and the row a recorded value hides under."""
+
+import re
+
+from .contract_screen import ATOM, NUMBER, as_float, center_y, element_text, plain
+
+# Bars and sticky footers sit in the screen's bottom band (from BOTTOM_BAND of its height down). A row a bar lies over
+# by more than OVER_ROW, or that the screen's foot cuts (rows are clipped to the screen), is in the tree but not all in
+# the picture. A row taller than TALL_ROW is a pane: no scroll brings all of it into view. Rows hidden under a top bar
+# are not counted: no stroke reliably starts inside the content under a header (TeamChat's pinned chips took both the
+# drag's and the glide's start, live, 18-19 Oct).
+BOTTOM_BAND, OVER_ROW, TALL_ROW, EDGE = .8, .005, .5, .005
+BAR_ROLES = frozenset({"NavigationBar", "TabBar", "Toolbar"})
+# A control this wide in the band is a bar (a sheet's "Request CityRideX" footer), and so is one of a line of BAR_LINE
+# controls spanning as much (a SwiftUI tab bar is only its buttons, centres within BAR_LINE_Y).
+BAR_WIDE, BAR_LINE, BAR_LINE_Y = .6, 3, .02
+# A name in a recorded value: two or more capitalized words ("Seaport Raw Bar", "Rowan Byrne").
+NAMES = re.compile(r"\b[A-Z][a-z]+(?: [A-Z][a-z]+)+")
+
+
+def value_atoms(value):
+    """What a screen row must hold to be a recorded value's source: its numbers, codes, times (ATOM) and names."""
+    return ATOM.findall(value or "") + NAMES.findall(value or "")
+
+
+def holds_atom(atom, text):
+    """Whether ``text`` (plain) holds ``atom``: a number as an equal number ("$1,200" in "1200.00"), else its text."""
+    if atom.startswith("$") or re.fullmatch(r"\d[\d,]*(?:\.\d+)?", atom):
+        number = as_float(atom.lstrip("$").strip())
+        return number is not None and any(abs((as_float(n) or -1e18) - number) < .005 for n in NUMBER.findall(text))
+    return plain(atom) in text
+
+
+def within(inner, outer, slack=.005):
+    ix, iy, iw, ih = inner
+    ox, oy, ow, oh = outer
+    return ix >= ox - slack and iy >= oy - slack and ix + iw <= ox + ow + slack and iy + ih <= oy + oh + slack
+
+
+def screen_bars(elements):
+    """[(index, bar)]: the bars and sticky footers of the screen's bottom band, by their place in ``elements``
+    (document order, back to front). A tab bar or toolbar, a control BAR_WIDE wide, or one of a line of controls as
+    wide; texts and pictures are content, never a bar."""
+    banded = [(index, element) for index, element in enumerate(elements)
+              if element.role not in ("StaticText", "Image") and element.rect[2] > 0 and element.rect[3] > 0
+              and element.rect[1] >= BOTTOM_BAND]
+    bars = []
+    for index, element in banded:
+        line = [other for _, other in banded if abs(center_y(other) - center_y(element)) <= BAR_LINE_Y]
+        if (element.role in BAR_ROLES or element.rect[2] >= BAR_WIDE or len(line) >= BAR_LINE
+                and max(o.rect[0] + o.rect[2] for o in line) - min(o.rect[0] for o in line) >= BAR_WIDE):
+            bars.append((index, element))
+    return bars
+
+
+def hidden_below(row, index, bars):
+    """Whether the screen's foot, or a bar of its bottom band (screen_bars), lies over part of ``row`` (at ``index`` in
+    the screen's elements). A tab bar or toolbar lies over any row not inside it; any other bar only over rows drawn
+    before it (its own texts come after it). iOSWorld multi-069 (18 Oct): "Comfort, $22.40" and its "$22.40" were
+    read from the tree under the ride sheet's "Request CityRideX" footer, and no picture of the run showed them."""
+    x, y, w, h = row.rect
+    if w <= 0 or h <= 0 or h > TALL_ROW:
+        return False
+    if y + h >= 1 - EDGE:
+        return True
+    return any(bar is not row and min(x + w, bx + bw) > max(x, bx) and min(y + h, by + bh) - max(y, by) > OVER_ROW
+               and (not within(row.rect, bar.rect) if bar.role in BAR_ROLES else at > index)
+               for at, bar in bars for bx, by, bw, bh in [bar.rect])
+
+
+def hidden_row(items, elements, step):
+    """The row of ``elements`` that holds a value marked at ``step`` when every row holding it is hidden below
+    (hidden_below), else None. The agent scrolls that row into view before it leaves the screen, so the run's frames
+    show what its answer cites."""
+    bars = None
+    for item in items:
+        if item.value_step != step or not item.value or item.kind == "REPORT" and item.status != "found":
+            continue
+        bars = bars if bars is not None else screen_bars(elements)
+        for atom in value_atoms(item.value):
+            rows = [(i, e) for i, e in enumerate(elements) if holds_atom(atom, plain(element_text(e)))]
+            if rows and all(hidden_below(row, i, bars) for i, row in rows):
+                return rows[0][1]
+    return None

mobile_agent/corpus/__init__.py

@@ -0,0 +1,26 @@
+"""The opt-in screen-to-action corpus, dark behind ``MOBSTER_EXPERIMENTAL_CORPUS`` (default off).
+
+Every completed run is a list of (screen, action) pairs on real iOS. With the person's opt-in (``consent``), the
+run listener writes one record per step to a local JSONL file (``store``): the screen as roles and hashed labels
+(``skeleton``), the action without its text, the app and version, the outcome. Nothing leaves the Mac: ``mobster
+corpus export`` writes a masked bundle for the person to read, and sharing is a later decision.
+
+With the flag off, or the opt-in off, the hooks return what they were given and no file is made.
+"""
+
+import os
+
+ENV_FLAG = "MOBSTER_EXPERIMENTAL_CORPUS"
+
+
+def enabled(env=None):
+    """Whether the corpus is switched on at all (``MOBSTER_EXPERIMENTAL_CORPUS=1``)."""
+    env = os.environ if env is None else env
+    return (env.get(ENV_FLAG) or "").strip().casefold() in ("1", "true", "yes", "on")
+
+
+def install(hooks):
+    """Add ``mobster corpus`` to the engine's commands, only with the flag on (harness_api._load calls this)."""
+    if enabled():
+        from . import cli
+        hooks.cli.append(cli)

mobile_agent/corpus/cli.py

@@ -0,0 +1,63 @@
+"""`mobster corpus` (only with MOBSTER_EXPERIMENTAL_CORPUS=1): the opt-in, what is stored, and an export to read.
+
+Added to the engine's commands by corpus.install as an extension: ``add_arguments(subs, commands)`` and
+``run(args, output)``, which returns an exit code (0 done, 1 refused, 2 usage) or None for another command.
+"""
+
+import json
+import sys
+
+from . import consent, store
+
+
+def add_arguments(subs, commands):
+    parser = subs.add_parser("corpus", prog="mobster corpus", help="keep and inspect Mobster's private run corpus",
+                             description="Keep a private record of each finished task's screens and actions on this "
+                                         "Mac, if you opt in, and read it. Nothing is uploaded.")
+    sub = parser.add_subparsers(dest="corpus_command", metavar="<subcommand>")
+    for name, help in (("status", "say whether the corpus is on"), ("off", "turn the corpus off"),
+                       ("stats", "count what is stored")):
+        sub.add_parser(name, help=help).add_argument("--json", action="store_true", help="print JSON")
+    on = sub.add_parser("on", help="show what is kept and turn the corpus on",
+                        description=consent.TEXT)
+    on.add_argument("--yes", action="store_true", help="answer yes to the question after the text is shown")
+    export = sub.add_parser("export", help="write the stored records to a file you can read")
+    export.add_argument("--out", required=True, metavar="FILE", help="a new file to write")
+    commands["corpus"] = parser
+
+
+def run(args, output):
+    if getattr(args, "command", None) != "corpus":
+        return None
+    word = getattr(args, "corpus_command", None) or "status"
+    as_json = getattr(args, "json", False)
+    try:
+        if word == "on":
+            print(consent.TEXT)
+            if not args.yes:
+                if not sys.stdin.isatty() or input("\nTurn it on? Type yes: ").strip().lower() != "yes":
+                    print("Not turned on.", file=sys.stderr)
+                    return 1
+            consent.set_on(True)
+            print("\nThe corpus is on. `mobster corpus off` turns it off.")
+        elif word == "off":
+            consent.set_on(False)
+            print("The corpus is off. What is stored stays until you delete the corpus folder.")
+        elif word == "status":
+            state = {"on": consent.on(), "folder": str(store.directory())}
+            print(json.dumps(state) if as_json else f"The corpus is {'on' if state['on'] else 'off'}.")
+        elif word == "stats":
+            data = store.stats()
+            print(json.dumps(data) if as_json else
+                  f"{data['records']} steps from {data['runs']} runs, {data['bytes']} of {data['capBytes']} bytes "
+                  f"in {data['files']} files, in {store.directory()}")
+        elif word == "export":
+            count = store.export(args.out)
+            print(f"Wrote {count} records to {args.out}. Open it to see exactly what is stored.")
+        else:
+            print("mobster corpus: use on, off, status, stats or export", file=sys.stderr)
+            return 2
+    except (store.CorpusError, OSError) as error:
+        print(f"mobster corpus: {error}", file=sys.stderr)
+        return 1
+    return 0

mobile_agent/corpus/consent.py

@@ -0,0 +1,55 @@
+"""The opt-in: off until the person has been shown ``TEXT`` and said yes, and off again when the text changes.
+
+The setting is ``corpus/consent.json`` in the data folder, which Mobster for Mac and ``mobster corpus on`` write the
+same way. No file, an unreadable file or an older text version is off.
+"""
+
+import json
+import time
+
+from ..paths import user_data_dir, write_text_atomic
+
+TEXT_VERSION = 1
+TITLE = "Help Mobster learn from your runs"
+TEXT = f"""{TITLE}
+
+When this is on, Mobster keeps a private record of each task that finishes, one line per step, on this Mac only.
+Nothing is uploaded or shared. You can read everything it keeps with `mobster corpus export`, and turn it off at
+any time.
+
+What a record holds:
+  - the shape of the screen: each control's kind (Button, Cell, TextField) and a one-way hash of its label, with
+    numbers folded; a label that looks like an email, phone number, handle or link is replaced by its kind
+  - the action: tap, type, swipe (not the text typed), and which control it hit
+  - the app's bundle id and version, whether the screen changed, and a random id for the run
+
+What it never holds: screenshots, text you typed, names, handles, numbers, messages, or any screen text. Steps in
+Passwords, Wallet and Health, on a screen with a password field, and in a task you redirected or that did not finish
+are not kept."""
+
+
+def _path():
+    return user_data_dir() / "corpus" / "consent.json"
+
+
+def on():
+    """True when the person opted in to the current ``TEXT``."""
+    try:
+        state = json.loads(_path().read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return False
+    return isinstance(state, dict) and state.get("optedIn") is True and state.get("textVersion") == TEXT_VERSION
+
+
+def set_on(value):
+    """Record the answer to ``TEXT``; turning it on is only ever done after the text was shown."""
+    state = {"optedIn": bool(value), "textVersion": TEXT_VERSION, "at": int(time.time())}
+    _path().parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    write_text_atomic(_path(), json.dumps(state))
+    _path().chmod(0o600)
+
+
+def active():
+    """The flag is on and the person opted in: the only state in which anything is captured or written."""
+    from . import enabled
+    return enabled() and on()

mobile_agent/corpus/listener.py

@@ -0,0 +1,76 @@
+"""The run listener: keeps a run's steps in memory and writes them only when the run finished done, unredirected.
+
+A step is kept only while ``consent.active()``; with the corpus off a run costs this listener nothing (no state,
+no file). The skeleton comes from the step itself (``StepRecord.corpus``, filled by frontier/hooks.py), so nothing
+here reads the phone.
+"""
+
+import hashlib
+import threading
+
+from ..errors import MobsterError
+from ..frontier.protocol import TYPED_OPS
+from . import consent, store
+from .skeleton import fingerprint
+
+DENY = frozenset({"com.apple.Passwords", "com.apple.Passbook", "com.apple.Health"})
+MAX_STEPS = 400
+
+
+class CorpusListener:
+    def __init__(self):
+        self._runs = {}
+        self._lock = threading.Lock()
+
+    def on_created(self, run, runtime):
+        if consent.active():
+            with self._lock:
+                self._runs[run.id] = {"steps": [], "redirected": False, "runtime": runtime}
+
+    def on_step(self, run, record):
+        captured = getattr(record, "corpus", None)
+        with self._lock:
+            held = self._runs.get(getattr(run, "id", None))
+            if held is None:
+                return
+            if getattr(record, "approval", None) == "redirected":
+                held["redirected"] = True
+            if (captured is None or record.app in DENY or not record.app or len(held["steps"]) >= MAX_STEPS
+                    or not consent.active()):
+                return
+            held["steps"].append((record, captured))
+
+    def on_finished(self, run, summary, private):
+        with self._lock:
+            held = self._runs.pop(getattr(run, "id", None), None)
+        done = (summary or {}).get("status") == "completed" and (summary or {}).get("outcome") == "done"
+        if held is None or held["redirected"] or not done or not held["steps"] or not consent.active():
+            return
+        rid = hashlib.sha256(f"corpus:{run.id}".encode()).hexdigest()[:16]
+        versions = {}
+        records = []
+        for number, (step, captured) in enumerate(held["steps"]):
+            if step.app not in versions:
+                versions[step.app] = _version(held["runtime"], step.app)
+            target = step.target if isinstance(step.target, dict) and captured["target"] is not None else None
+            nodes = captured["nodes"]
+            index = captured["target"]
+            records.append({
+                "v": store.SCHEMA, "run": rid, "step": number, "app": step.app, "version": versions[step.app],
+                "screen": {"fp": fingerprint(nodes), "nodes": nodes},
+                "action": {"op": step.op, "typed": step.op in TYPED_OPS,
+                           "target": {"i": index, "role": nodes[index][0], "label": nodes[index][1]}
+                           if target is not None else None},
+                "outcome": {"changed": bool(step.changed), "run": "done"}})
+        try:
+            store.append(records)
+        except (OSError, store.CorpusError, MobsterError):
+            pass  # the corpus is a courtesy: a full disk or a bad folder never touches a run
+
+
+def _version(runtime, bundle):
+    from ..packs.core import app_version
+    try:
+        return app_version(runtime, bundle)
+    except Exception:  # noqa: BLE001 -- no version is a valid answer
+        return None

mobile_agent/corpus/register.py

@@ -0,0 +1,9 @@
+"""Track registration: the run listener, only with MOBSTER_EXPERIMENTAL_CORPUS on."""
+
+from . import enabled
+
+
+def register(api):
+    if enabled():
+        from .listener import CorpusListener
+        api.register_run_listener(CorpusListener())

mobile_agent/corpus/skeleton.py

@@ -0,0 +1,72 @@
+"""A screen as a skeleton: each control's role and a one-way hash of its label. No screen text survives.
+
+Labels are the risk: a label can be a name, a handle, a number or a message. So a label is
+- empty -> "";
+- long (prose: a message, a caption, a title) -> "~long";
+- an email, phone number, handle or link, or anything secret_filter would mask -> "~email", "~number", "~handle",
+  "~link", "~secret";
+- otherwise folded (case, digit runs to "#") and hashed to 10 hex digits (compiled_read/routes.py's digest), so
+  "General" is the same token on every Mac and "3 new messages" equals "5 new messages".
+A short name ("Sam") is a short label and hashes to a token anyone can test a guess against: why the corpus stays
+local until sharing is decided.
+"""
+
+import hashlib
+import re
+import unicodedata
+
+from ..secret_filter import is_secret
+
+MAX_NODES = 80
+MAX_LABEL = 48
+EMAIL = re.compile(r"\S+@\S+\.\S+")
+HANDLE = re.compile(r"(?<!\w)@\w{2,}")
+LINK = re.compile(r"(?i)https?://|www\.|\b[\w-]+\.(?:com|org|net|io|app|dev|me)\b")
+DIGITS = re.compile(r"\d+")
+PHONE = re.compile(r"(?<!\w)\+?\d[\d\s().-]{5,}\d")
+ROLE = re.compile(r"[A-Za-z]{1,40}")
+TOKEN = re.compile(r"[0-9a-f]{10}|~(?:long|email|number|handle|link|secret)|")
+# Roles of a field that holds a password: the whole step is dropped, not masked.
+SECURE = "secure"
+
+
+def label_token(label):
+    text = unicodedata.normalize("NFKC", label or "").strip()
+    if not text:
+        return ""
+    if len(text) > MAX_LABEL:
+        return "~long"
+    if is_secret(text):
+        return "~secret"
+    for name, pattern in (("email", EMAIL), ("link", LINK), ("handle", HANDLE), ("number", PHONE)):
+        if pattern.search(text):
+            return "~" + name
+    if sum(map(len, DIGITS.findall(text))) >= 7:
+        return "~number"
+    return hashlib.sha1(DIGITS.sub("#", text.casefold()).encode()).hexdigest()[:10]
+
+
+def _role(role):
+    return role if ROLE.fullmatch(role or "") else "Other"
+
+
+def capture(snapshot, element):
+    """``{"nodes": [[role, token], ...], "target": index or None}`` for a screen and the control acted on; ``None``
+    when nothing may be kept of it: the corpus is off, or the screen has a secure field."""
+    from .consent import active
+    if not active():
+        return None
+    elements = list(getattr(snapshot, "elements", None) or ())
+    if any(SECURE in str(getattr(item, "role", "")).casefold() for item in elements):
+        return None
+    nodes, target = [], None
+    for item in elements[:MAX_NODES]:
+        if element is not None and target is None and getattr(item, "id", None) == element.id:
+            target = len(nodes)
+        nodes.append([_role(str(getattr(item, "role", ""))), label_token(getattr(item, "label", ""))])
+    return {"nodes": nodes, "target": target}
+
+
+def fingerprint(nodes):
+    """16 hex digits for a skeleton: the same screen shape on any Mac gives the same value."""
+    return hashlib.sha1(repr(sorted(map(tuple, nodes))).encode()).hexdigest()[:16]

mobile_agent/corpus/store.py

@@ -0,0 +1,143 @@
+"""The corpus on disk: ``corpus/records.jsonl`` in the data folder (0700 folder, 0600 files), capped and rotated.
+
+``records.jsonl`` takes appends until it reaches FILE_BYTES; then it becomes ``records.1.jsonl``, that one
+``records.2.jsonl``, and the oldest is deleted. The corpus never holds more than FILES * FILE_BYTES.
+``sanitize`` is the one gate: a record is written, and exported, only if every field has the shape below.
+"""
+
+import fcntl
+import json
+import os
+import re
+import stat
+from pathlib import Path
+
+from ..paths import user_data_dir
+from ..secret_filter import redact_deep
+from .skeleton import MAX_NODES, ROLE, TOKEN
+
+FILE_BYTES = 4_000_000
+FILES = 3
+SCHEMA = 1
+OPS = re.compile(r"[A-Z_]{2,24}")
+HASH = re.compile(r"[0-9a-f]{16}")
+BUNDLE = re.compile(r"[A-Za-z0-9.-]{1,155}")
+VERSION = re.compile(r"[0-9A-Za-z.+-]{1,40}")
+
+
+class CorpusError(Exception):
+    pass
+
+
+def directory():
+    return user_data_dir() / "corpus"
+
+
+def _private(path):
+    info = path.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid():
+        raise CorpusError("the corpus folder must be a folder you own")
+
+
+def _files():
+    root = directory()
+    return [root / ("records.jsonl" if n == 0 else f"records.{n}.jsonl") for n in range(FILES)]
+
+
+def _node(node):
+    return (isinstance(node, list) and len(node) == 2 and isinstance(node[0], str) and ROLE.fullmatch(node[0])
+            and isinstance(node[1], str) and TOKEN.fullmatch(node[1]))
+
+
+def sanitize(record):
+    """``record`` as the schema allows it, or None. Every string is a fixed-shape token or passes secret_filter."""
+    try:
+        screen, action, outcome = record["screen"], record["action"], record["outcome"]
+        target = action["target"]
+        ok = (record["v"] == SCHEMA and HASH.fullmatch(record["run"]) and isinstance(record["step"], int)
+              and 0 <= record["step"] < 10_000 and BUNDLE.fullmatch(record["app"])
+              and (record["version"] is None or VERSION.fullmatch(record["version"]))
+              and HASH.fullmatch(screen["fp"]) and isinstance(screen["nodes"], list)
+              and len(screen["nodes"]) <= MAX_NODES and all(_node(n) for n in screen["nodes"])
+              and OPS.fullmatch(action["op"]) and isinstance(action["typed"], bool)
+              and (target is None or (isinstance(target["i"], int) and 0 <= target["i"] < len(screen["nodes"])
+                                      and _node([target["role"], target["label"]])))
+              and isinstance(outcome["changed"], bool) and outcome["run"] == "done")
+    except (KeyError, TypeError):
+        return None
+    return record if ok and redact_deep(record) == record else None
+
+
+def append(records):
+    """Append ``records`` (already sanitized) as one write; returns how many were written."""
+    lines = "".join(json.dumps(r, separators=(",", ":")) + "\n" for r in records if sanitize(r))
+    if not lines:
+        return 0
+    root = directory()
+    root.parent.mkdir(parents=True, exist_ok=True)
+    root.mkdir(mode=0o700, exist_ok=True)
+    _private(root)
+    files = _files()
+    lock = os.open(root / ".lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+    try:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        if files[0].exists() and files[0].stat().st_size + len(lines) > FILE_BYTES:
+            files[-1].unlink(missing_ok=True)
+            for newer, older in reversed(list(zip(files, files[1:]))):
+                if newer.exists():
+                    os.replace(newer, older)
+        fd = os.open(files[0], os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+        with os.fdopen(fd, "a", encoding="utf-8") as handle:
+            handle.write(lines)
+            handle.flush()
+            os.fsync(handle.fileno())
+    finally:
+        os.close(lock)
+    return lines.count("\n")
+
+
+def read():
+    """Every stored record that still passes ``sanitize``, oldest file first."""
+    for path in reversed(_files()):
+        try:
+            handle = open(path, encoding="utf-8")
+        except OSError:
+            continue
+        with handle:
+            for line in handle:
+                try:
+                    record = sanitize(json.loads(line))
+                except ValueError:
+                    continue
+                if record:
+                    yield record
+
+
+def stats():
+    """Counts for ``mobster corpus stats``: records, runs, steps per app, bytes, files."""
+    records, runs, apps = 0, set(), {}
+    for record in read():
+        records += 1
+        runs.add(record["run"])
+        apps[record["app"]] = apps.get(record["app"], 0) + 1
+    present = [p for p in _files() if p.exists()]
+    return {"records": records, "runs": len(runs), "bytes": sum(p.stat().st_size for p in present),
+            "files": len(present), "capBytes": FILE_BYTES * FILES, "apps": dict(sorted(apps.items()))}
+
+
+def export(out):
+    """Write the masked bundle to ``out`` (a new file, 0600): a header line, then one record per line."""
+    out = Path(out).expanduser()
+    records = list(read())
+    header = {"bundle": "mobster-corpus", "v": SCHEMA, "records": len(records),
+              "note": "roles, hashed labels and actions only; no screen text, no typed text"}
+    try:
+        fd = os.open(out, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    except FileExistsError:
+        raise CorpusError(f"{out} exists; choose a path that doesn't") from None
+    except OSError as error:
+        raise CorpusError(f"{out} can't be written ({error.strerror})") from None
+    with os.fdopen(fd, "w", encoding="utf-8") as handle:
+        for item in [header, *records]:
+            handle.write(json.dumps(item, separators=(",", ":")) + "\n")
+    return len(records)