mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/mac_tools.py

@@ -0,0 +1,223 @@
+"""Finding the Mac's own tools, and asking the Mac short questions."""
+
+import functools
+import os
+import shutil
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+from .devices import LOOPBACK, UDID_PATTERN
+
+TOOL_DIRS = ("/opt/homebrew/bin", "/usr/local/bin", "/usr/bin", "/Applications/Xcode.app/Contents/Developer/usr/bin")
+# The iPhone connection tools: libimobiledevice's and libusbmuxd's. Mobster for Mac carries its own build of them
+# (desktop/scripts/build-iphone-tools.sh), found before any Homebrew copy, so nobody needs Homebrew or Terminal.
+IPHONE_TOOLS = ("idevice_id", "ideviceinfo", "ideviceinstaller", "iproxy", "afcclient", "idevicesyslog",
+                "idevicecrashreport")
+# /usr/bin/xcodebuild and /usr/bin/git exist on every Mac: they are stubs that forward to the active
+# developer directory (and fail, or open an install dialog, without one). They never count as Xcode or
+# git: both come from the Xcode app itself, git also from the Command Line Tools or Homebrew.
+XCODE_SHIMS = frozenset({"xcodebuild", "git"})
+CLT_DIR = "/Library/Developer/CommandLineTools"
+XCODE_APP_DIRS = ("/Applications", str(Path.home() / "Applications"))
+GIT_DIRS = ("/opt/homebrew/bin", "/usr/local/bin", CLT_DIR + "/usr/bin")
+# Xcode's state takes a few short xcodebuild calls: a working Xcode is re-checked rarely, one that
+# needs fixing often enough that the fix shows up on its own.
+XCODE_OK_TTL, XCODE_RETRY_TTL = 300.0, 10.0
+WDA_PORT, VIDEO_PORT = 8100, 9100
+
+
+def executable(directory, name):
+    path = Path(directory) / name
+    return str(path) if path.is_file() and os.access(path, os.X_OK) else None
+
+
+def bundled_tools_dir():
+    override = os.environ.get("MOBSTER_IPHONE_TOOLS")
+    if override:
+        return Path(override)
+    frozen = getattr(sys, "_MEIPASS", None)
+    if not frozen:
+        return None
+    for resources in (Path(frozen).parent / "Resources", Path(sys.executable).resolve().parent.parent / "Resources"):
+        if (resources / "iphone-tools" / "bin").is_dir():
+            return resources / "iphone-tools" / "bin"
+    return None
+
+
+def bundled_tool(name):
+    """The app's own copy of an iPhone connection tool, or None."""
+    directory = bundled_tools_dir() if name in IPHONE_TOOLS else None
+    return executable(directory, name) if directory else None
+
+
+def tool(name):
+    if name in XCODE_SHIMS:
+        return xcode_tool(name)
+    if path := bundled_tool(name):
+        return path
+    for directory in TOOL_DIRS:
+        if path := executable(directory, name):
+            return path
+    return shutil.which(name)
+
+
+def is_xcode(path):
+    """A developer directory inside an Xcode app (not the Command Line Tools)."""
+    path = Path(path)
+    return (path.name == "Developer" and path.parent.name == "Contents" and path.parent.parent.suffix == ".app"
+            and executable(path / "usr" / "bin", "xcodebuild") is not None)
+
+
+def developer_dir():
+    selected = (run(["/usr/bin/xcode-select", "-p"], timeout=5) or "").strip().rstrip("/") or None
+    if selected and is_xcode(selected):
+        return selected, selected
+    for root in XCODE_APP_DIRS:
+        try:
+            apps = sorted(Path(root).glob("Xcode*.app"), key=lambda app: (app.name != "Xcode.app", app.name))
+        except OSError:
+            continue
+        for app in apps:
+            if is_xcode(app / "Contents" / "Developer"):
+                return str(app / "Contents" / "Developer"), selected
+    return None, selected
+
+
+# A shell's compiler settings that clang and xcodebuild read from the environment. Meant for building for this Mac
+# (Homebrew, conda), they point an iPhone or simulator build at the Mac's headers and SDK: on 17 Oct a shell with
+# CPATH=/usr/local/include failed the helper's first build ("include location '/usr/local/include' is unsafe for
+# cross-compilation", -Werror). Xcode's builds never take them.
+HOST_BUILD_ENV = ("CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH", "OBJCPLUS_INCLUDE_PATH",
+                  "LIBRARY_PATH", "SDKROOT", "CFLAGS", "CXXFLAGS", "OBJCFLAGS", "CPPFLAGS", "LDFLAGS")
+
+
+def xcode_env(developer=None, base=None):
+    env = {key: value for key, value in (os.environ if base is None else base).items() if key not in HOST_BUILD_ENV}
+    if developer:
+        env["DEVELOPER_DIR"] = developer
+    return env
+
+
+# In place of asking for `sudo xcode-select -s`: only this process's environment changes, never the Mac's setting.
+def adopt_xcode(environ=None):
+    environ = os.environ if environ is None else environ
+    if environ.get("DEVELOPER_DIR"):
+        return environ["DEVELOPER_DIR"]
+    xcode, selected = developer_dir()
+    if xcode and selected != xcode:
+        environ["DEVELOPER_DIR"] = xcode
+    return xcode
+
+
+def xcode_tool(name, developer=...):
+    developer = developer_dir()[0] if developer is ... else developer
+    directories = (([str(Path(developer) / "usr" / "bin")] if developer else [])
+                   + (list(GIT_DIRS) if name == "git" else []))
+    return next((path for directory in directories if (path := executable(directory, name))), None)
+
+
+def execute(command, env=None, timeout=20):
+    """(exit code, stdout and stderr) of a short command; (None, "") if it could not run."""
+    try:
+        completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout, env=env,
+                                   stdin=subprocess.DEVNULL)
+    except (OSError, subprocess.SubprocessError):
+        return None, ""
+    return completed.returncode, (completed.stdout or "") + (completed.stderr or "")
+
+
+# The Mac's xcode-select choice is never a problem: Mobster runs Xcode's tools with DEVELOPER_DIR (adopt_xcode).
+def xcode_status():
+    xcode, selected = developer_dir()
+    status = {"state": "ok", "app": str(Path(xcode).parent.parent) if xcode else None, "developer_dir": xcode,
+              "selected": selected, "version": None, "problems": [], "fixes": []}
+    problems, fixes = status["problems"], status["fixes"]
+    if not xcode:
+        problems.append("missing")
+    else:
+        xcodebuild, env = str(Path(xcode) / "usr" / "bin" / "xcodebuild"), {**os.environ, "DEVELOPER_DIR": xcode}
+        code, output = execute([xcodebuild, "-version"], env)
+        if code == 0:
+            status["version"] = (output.strip().splitlines() or [None])[0]
+            if execute([xcodebuild, "-checkFirstLaunchStatus"], env)[0] not in (0, None):
+                problems.append("first_launch")
+                fixes.append("sudo xcodebuild -runFirstLaunch")
+            else:
+                code, sdks = execute([xcodebuild, "-showsdks"], env, timeout=30)
+                if code == 0 and "iphoneos" not in sdks:
+                    problems.append("no_ios")
+                    fixes.append("xcodebuild -downloadPlatform iOS")
+        elif "license" in output.lower():
+            problems.append("license")
+            fixes.append("sudo xcodebuild -license accept")
+        else:
+            problems.append("broken")
+    status["state"] = problems[0] if problems else "ok"
+    return status
+
+
+def count_iphones(items, simulated=False):
+    """iPhones in a system_profiler USB tree; a virtual one on a simulated bus doesn't count."""
+    total = 0
+    for item in items if isinstance(items, list) else ():
+        if not isinstance(item, dict):
+            continue
+        on_simulated = simulated or item.get("USBKeyHardwareType") == "Simulated"
+        name = str(item.get("_name") or "")
+        vendor = str(item.get("USBDeviceKeyVendorID") or item.get("vendor_id") or "").lower()
+        if name.startswith("iPhone") and "virtual" not in name.lower() and "0x05ac" in vendor and not on_simulated:
+            total += 1
+        total += count_iphones(item.get("_items"), on_simulated)
+    return total
+
+
+def run(command, timeout=10):
+    """stdout of a short command, or None if it failed."""
+    try:
+        completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return completed.stdout if completed.returncode == 0 else None
+
+
+# How long one USB listing answers ``usb_attached``: a pulled cable is known within this many seconds.
+ATTACHED_TTL = 2.0
+_attached_cache = {"at": None, "udids": None}
+_attached_lock = threading.Lock()
+
+
+def usb_attached(udid, now=None, ttl=ATTACHED_TTL):
+    if not isinstance(udid, str) or not UDID_PATTERN.fullmatch(udid):
+        return None
+    now = time.monotonic() if now is None else now
+    with _attached_lock:
+        at, udids = _attached_cache["at"], _attached_cache["udids"]
+        if at is None or now - at >= ttl:
+            listing = tool("idevice_id")
+            output = run([listing, "-l"], timeout=3) if listing else None
+            udids = None if output is None else {line.strip().upper() for line in output.splitlines() if line.strip()}
+            _attached_cache.update(at=now, udids=udids)
+    return None if udids is None else udid.upper() in udids
+
+
+@functools.lru_cache(maxsize=4)
+def iproxy_binds_loopback(path):
+    """Whether this iproxy takes -s; an older one cannot be kept off the LAN."""
+    try:
+        completed = subprocess.run([path, "-h"], capture_output=True, text=True, timeout=5)
+    except (OSError, subprocess.SubprocessError):
+        return False
+    return "--source" in completed.stdout + completed.stderr
+
+
+def relay_command(udid, wda_port=WDA_PORT, video_port=VIDEO_PORT):
+    iproxy = tool("iproxy")
+    if not iproxy:
+        raise LookupError("The iPhone connection tools aren't installed. Open Setup › Get your Mac ready.")
+    if not iproxy_binds_loopback(iproxy):
+        raise LookupError("This iproxy can't be limited to this Mac. Update it: "
+                          "brew upgrade libimobiledevice libusbmuxd")
+    return [iproxy, "-s", LOOPBACK, "-u", udid, f"{wda_port}:{WDA_PORT}", f"{video_port}:{VIDEO_PORT}"]

mobile_agent/manual_control.py

@@ -0,0 +1,143 @@
+"""Direct control of the phone from the dashboard's live view; the server refuses it while a task holds the phone."""
+
+import math
+import threading
+import time
+
+from .drivers import WDA
+from .state import validate_input_text
+from .wda.gestures import WDA_TAP_HOLD_MS
+
+BUTTONS = {"home": "home", "volume_up": "volumeUp", "volume_down": "volumeDown"}
+MAX_HOLD_MS = 3000
+SWIPE_MS = (40, 3000)
+SIZE_TTL_SECONDS = 30
+
+
+def _unit(value, name):
+    if type(value) not in (int, float) or not 0 <= value <= 1:
+        raise ValueError(f"{name} must be a number from 0 to 1")
+    return float(value)
+
+
+def parse_command(body):
+    """A validated gesture, or ValueError. Unknown fields are refused, never ignored."""
+    if not isinstance(body, dict):
+        raise ValueError("A control command must be an object")
+    kind = body.get("action")
+    allowed = {"tap": {"x", "y", "holdMs"}, "swipe": {"x1", "y1", "x2", "y2", "durationMs"},
+               "type": {"text", "submit"}, "button": {"name"}}
+    if kind not in allowed:
+        raise ValueError("action must be tap, swipe, type or button")
+    extra = set(body) - allowed[kind] - {"action"}
+    if extra:
+        raise ValueError(f"Unsupported field: {sorted(extra)[0]}")
+    if kind == "tap":
+        hold = body.get("holdMs", WDA_TAP_HOLD_MS)
+        if type(hold) is not int or not 0 <= hold <= MAX_HOLD_MS:
+            raise ValueError(f"holdMs must be 0 to {MAX_HOLD_MS}")
+        return {"action": "tap", "x": _unit(body.get("x"), "x"), "y": _unit(body.get("y"), "y"), "holdMs": hold}
+    if kind == "swipe":
+        duration = body.get("durationMs", 250)
+        if type(duration) is not int or not SWIPE_MS[0] <= duration <= SWIPE_MS[1]:
+            raise ValueError(f"durationMs must be {SWIPE_MS[0]} to {SWIPE_MS[1]}")
+        return {"action": "swipe", **{k: _unit(body.get(k), k) for k in ("x1", "y1", "x2", "y2")},
+                "durationMs": duration}
+    if kind == "type":
+        submit = body.get("submit", False)
+        if type(submit) is not bool:
+            raise ValueError("submit must be true or false")
+        return {"action": "type", "text": validate_input_text(body.get("text")), "submit": submit}
+    if body.get("name") not in BUTTONS:
+        raise ValueError("name must be home, volume_up or volume_down")
+    return {"action": "button", "name": body["name"]}
+
+
+class ManualControl:
+    def __init__(self, wda_url, session_provider):
+        self.wda_url, self.session_provider = wda_url, session_provider
+        self.lock = threading.Lock()
+        self.driver = None
+        self.session = None
+        self.size = None  # (width, height, read_at)
+
+    def _driver(self):
+        session = self.session_provider()
+        if not session:
+            raise ConnectionError("Your iPhone is not connected")
+        if self.driver is None or session != self.session:
+            self.close()
+            self.driver, self.session, self.size = WDA(self.wda_url, session), session, None
+        return self.driver
+
+    def _points(self, driver):
+        """The screen in points. /wda/screen measures through SpringBoard; /window/size
+        goes through the "active" app, which can be a transient system overlay (measured
+        on iOS 26: an 8-10 s stall, then a stale-element error), so it is only a fallback."""
+        if self.size is None or time.monotonic() - self.size[2] > SIZE_TTL_SECONDS:
+            size = None
+            try:
+                screen = WDA.response_value(driver.http.request("GET", "/wda/screen", None, 3))
+                size = screen.get("screenSize") if isinstance(screen, dict) else None
+            except Exception:
+                size = None
+            if not self._valid(size):
+                size = driver.call("GET", "/window/size", timeout=3)
+            if not self._valid(size):
+                raise ConnectionError("Your iPhone did not report its screen size")
+            self.size = (float(size["width"]), float(size["height"]), time.monotonic())
+        return self.size[:2]
+
+    @staticmethod
+    def _valid(size):
+        return isinstance(size, dict) and all(
+            type(size.get(k)) in (int, float) and math.isfinite(size[k]) and size[k] > 0 for k in ("width", "height"))
+
+    def perform(self, body, teach=None):
+        """Send one gesture; with ``teach``, the screen is read first and nothing is sent when it can't be."""
+        command = parse_command(body)
+        with self.lock:
+            driver = self._driver()
+            before = teach.read(driver) if teach is not None else None
+            try:
+                self._dispatch(driver, command)
+            except Exception:
+                # A dropped session is re-resolved on the next gesture.
+                self.close()
+                raise
+            if teach is not None:
+                teach.record(command, before)
+        return command["action"]
+
+    def observe(self, timeout=8):
+        """The screen now (a teach session's end screen)."""
+        with self.lock:
+            return self._driver().observe(timeout=timeout)
+
+    def _dispatch(self, driver, command):
+        kind = command["action"]
+        if kind == "button":
+            driver.call("POST", "/wda/pressButton", {"name": BUTTONS[command["name"]]}, timeout=5)
+            return
+        if kind == "type":
+            keys = [command["text"], "\n"] if command["submit"] else [command["text"]]
+            driver.call("POST", "/wda/keys", {"value": keys}, timeout=10)
+            return
+        width, height = self._points(driver)
+        if kind == "tap":
+            x, y = round(command["x"] * width), round(command["y"] * height)
+            steps = [{"type": "pointerMove", "duration": 0, "x": x, "y": y}, {"type": "pointerDown"},
+                     {"type": "pause", "duration": command["holdMs"]}, {"type": "pointerUp"}]
+        else:
+            steps = [{"type": "pointerMove", "duration": 0, "x": round(command["x1"] * width),
+                      "y": round(command["y1"] * height)}, {"type": "pointerDown"},
+                     {"type": "pointerMove", "duration": command["durationMs"], "x": round(command["x2"] * width),
+                      "y": round(command["y2"] * height)}, {"type": "pointerUp"}]
+        driver.call("POST", "/actions", {"actions": [{"type": "pointer", "id": "finger",
+                                                      "parameters": {"pointerType": "touch"}, "actions": steps}]},
+                    timeout=10 + (command.get("holdMs", 0) + command.get("durationMs", 0)) / 1000)
+
+    def close(self):
+        driver, self.driver, self.session = self.driver, None, None
+        if driver is not None:
+            driver.close()

mobile_agent/maps.py

@@ -0,0 +1,149 @@
+"""Places on a map, for a result's ``map`` block and a table's Map view (docs/results.mdx).
+
+``snapshot(places, ...)`` draws them on an Apple Maps snapshot made on this Mac by the ``map`` helper
+(helper_map.swift: MapKit off screen, no window): {"image": PNG bytes at 2x, "width", "height" (points), "places":
+[{"id", "lat", "lon", "x", "y"}], "missing": [ids]}. A place that has no coordinates is found by its address, then
+by its name near ``near``, with Apple's geocoder. The page draws its own numbered pins over the picture (x and y are
+where each one goes), so a pin is a button that names its place and says where it is.
+
+Snapshots are kept on disk by what they show (the places, the appearance and the size), so a result opened again
+draws at once and asks Apple's geocoder nothing. What reaches the helper is a place's name and address or
+coordinates: the same words the result already shows.
+
+``places_of(block)``: a ``map`` block's places, or a table's rows as places (a Name column, an Address or Location
+column, or Latitude and Longitude columns).
+"""
+
+import hashlib
+import json
+import re
+import subprocess
+import tempfile
+import threading
+from pathlib import Path
+
+SECONDS = 45
+KEEP = 200
+MAX_PLACES = 30
+_lock = threading.BoundedSemaphore(2)
+
+NAME = re.compile(r"^(name|place|title|restaurant|store|shop|venue|business|cafe|café|spot)$", re.I)
+ADDRESS = re.compile(r"^(address|location|where|street|addr\.?)$", re.I)
+LAT = re.compile(r"^(lat|latitude)$", re.I)
+LON = re.compile(r"^(lon|lng|long|longitude)$", re.I)
+
+
+class MapError(RuntimeError):
+    """The map couldn't be drawn; the message says why, in plain words."""
+
+
+def _column(columns, pattern):
+    return next((c["key"] for c in columns if pattern.match(str(c.get("label") or "").strip())), None)
+
+
+def places_of(block):
+    """[{"id", "name", "address"?, "lat"?, "lon"?}] for a map block or a table with places, else []."""
+    if not isinstance(block, dict):
+        return []
+    if block.get("type") == "map":
+        return [{key: place[key] for key in ("id", "name", "address", "lat", "lon") if key in place}
+                for place in block.get("places") or ()][:MAX_PLACES]
+    if block.get("type") != "table":
+        return []
+    columns = block.get("columns") or []
+    address, lat, lon = _column(columns, ADDRESS), _column(columns, LAT), _column(columns, LON)
+    if address is None and (lat is None or lon is None):
+        return []
+    name = _column(columns, NAME) or next((c["key"] for c in columns if c["key"] not in (address, lat, lon)), None)
+    places = []
+    for index, row in enumerate(block.get("rows") or ()):
+        cells = row.get("cells") or {}
+        place = {"id": f"r{index}", "name": str(cells.get(name) or f"Place {index + 1}")[:120]}
+        if address is not None and isinstance(cells.get(address), str) and cells[address].strip():
+            place["address"] = cells[address].strip()[:200]
+        if lat is not None and lon is not None:
+            try:
+                y, x = float(cells.get(lat)), float(cells.get(lon))
+                if -90 <= y <= 90 and -180 <= x <= 180:
+                    place["lat"], place["lon"] = y, x
+            except (TypeError, ValueError):
+                pass
+        if "address" in place or "lat" in place:
+            places.append(place)
+        if len(places) >= MAX_PLACES:
+            break
+    return places
+
+
+def _cache_dir():
+    from .paths import user_data_dir
+    folder = Path(user_data_dir()) / "cache" / "maps"
+    folder.mkdir(parents=True, exist_ok=True, mode=0o700)
+    return folder
+
+
+def _prune(folder):
+    files = sorted(folder.glob("*.png"), key=lambda path: path.stat().st_mtime)
+    for old in files[:-KEEP]:
+        for path in (old, old.with_suffix(".json")):
+            path.unlink(missing_ok=True)
+
+
+def snapshot(places, *, near=None, dark=False, width=720, height=360, helper=None, cache_dir=None,
+             timeout=SECONDS):
+    """The places on a map (see the module docstring). MapError when there's nothing to draw or no helper."""
+    places = [p for p in places or () if isinstance(p, dict) and p.get("name")][:MAX_PLACES]
+    if not places:
+        raise MapError("There are no places to show on a map.")
+    width, height = max(200, min(1200, int(width))), max(120, min(900, int(height)))
+    request = {"places": places, "near": near, "width": width, "height": height, "dark": bool(dark)}
+    key = hashlib.sha256(json.dumps(request, sort_keys=True).encode()).hexdigest()[:24]
+    folder = Path(cache_dir) if cache_dir is not None else _cache_dir()
+    image, meta = folder / f"{key}.png", folder / f"{key}.json"
+    if image.is_file() and meta.is_file():
+        try:
+            return {**json.loads(meta.read_text()), "image": image.read_bytes()}
+        except (OSError, ValueError):
+            pass
+    if helper is None:
+        from . import native_helpers
+        helper = native_helpers.find("map")
+        if helper is None:
+            try:
+                helper = native_helpers.build("map")
+            except (LookupError, OSError, subprocess.SubprocessError) as error:
+                raise MapError("Mobster can't draw maps on this Mac: " + str(error)) from None
+    if not _lock.acquire(timeout=timeout):
+        raise MapError("Mobster is drawing other maps. Try again in a moment.")
+    try:
+        with tempfile.TemporaryDirectory(prefix="mobster-map-") as work:
+            source, target = Path(work) / "places.json", Path(work) / "map.png"
+            source.write_text(json.dumps(request), encoding="utf-8")
+            try:
+                done = subprocess.run([str(helper), str(source), str(target)], capture_output=True, text=True,
+                                      timeout=timeout)
+            except subprocess.TimeoutExpired:
+                raise MapError("Mobster took too long to draw the map. Try again.") from None
+            except OSError as error:
+                raise MapError(f"Mobster couldn't draw the map: {error}") from None
+            if done.returncode != 0 or not target.is_file():
+                lines = (done.stderr or "").strip().splitlines()
+                said = next((l for l in reversed(lines) if l.startswith("Mobster")), "Mobster couldn't draw the map")
+                raise MapError(said if said.endswith(".") else said + ".")
+            try:
+                answer = json.loads((done.stdout or "").strip().splitlines()[-1])
+            except (ValueError, IndexError):
+                raise MapError("Mobster couldn't read where the places are.") from None
+            data = target.read_bytes()
+    finally:
+        _lock.release()
+    out = {"width": answer.get("width", width), "height": answer.get("height", height),
+           "places": [p for p in answer.get("places") or () if isinstance(p, dict)],
+           "missing": [m for m in answer.get("missing") or () if isinstance(m, str)]}
+    try:
+        image.write_bytes(data)
+        meta.write_text(json.dumps(out))
+        _prune(folder)
+    except OSError:
+        pass
+    return {**out, "image": data}

mobile_agent/marks.py

@@ -0,0 +1,92 @@
+"""Set-of-Mark: numbered boxes on the screenshot, one per element a decision model may pick.
+
+A decision model answers "which mark" and "which gesture"; code turns the mark back into the exact element (its
+AX frame's centre when it taps), so the model never writes a coordinate.
+
+``Mark`` keeps the element; ``mark_lines`` is the same list as text, for a text-only decider (Jev) and so both
+deciders get identical options.
+"""
+
+import io
+from dataclasses import dataclass
+
+TARGET_ROLES = frozenset({"Button", "Cell", "Link", "TextField", "SearchField", "SecureTextField", "TextView",
+                          "Switch", "Toggle", "Tab", "TabBar", "MenuItem", "Image", "Icon", "SegmentedControl",
+                          "Key", "PickerWheel", "Slider", "Stepper", "Other", "StaticText"})
+MAX_MARKS = 120
+
+
+@dataclass(frozen=True)
+class Mark:
+    id: str
+    element: object
+    rect: tuple
+    label: str
+    role: str
+
+    @property
+    def center(self):
+        x, y, w, h = self.rect
+        return x + w / 2, y + h / 2
+
+
+def element_marks(elements, limit=MAX_MARKS):
+    """Marks for the targetable elements on screen, top to bottom, ids "1", "2"...; at most ``limit``."""
+    out = []
+    for element in elements:
+        if len(out) >= limit:
+            break
+        x, y, w, h = element.rect
+        if w <= 0 or h <= 0 or y >= 1 or y + h <= 0:
+            continue
+        if element.role not in TARGET_ROLES and not getattr(element, "editable", False):
+            continue
+        out.append(Mark(str(len(out) + 1), element, tuple(element.rect), (element.label or "")[:120], element.role))
+    return out
+
+
+def mark_lines(marks):
+    """The marks as text: ``7 Button "Search" @4,14,92,5`` (frame in percent)."""
+    lines = []
+    for mark in marks:
+        x, y, w, h = (round(v * 100) for v in mark.rect)
+        label = f'"{mark.label}"' if mark.label else "(no label)"
+        value = getattr(mark.element, "value", "")
+        value = f" value={str(value)[:40]!r}" if value and value != mark.label else ""
+        lines.append(f"{mark.id} {mark.role} {label}{value} @{x},{y},{w},{h}")
+    return lines
+
+
+def options(marks, *, abstain=True):
+    """A choice question's options for these marks (value = mark id), plus "none"."""
+    out = {mark.id: line for mark, line in zip(marks, mark_lines(marks))}
+    if abstain:
+        out["none"] = "None of the marked elements is right"
+    return out
+
+
+def draw(jpeg, marks, side=None):
+    """The screenshot (JPEG bytes) with each mark's box and number drawn; JPEG bytes back."""
+    from PIL import Image, ImageDraw, ImageFont
+    image = Image.open(io.BytesIO(jpeg)).convert("RGB")
+    if side and image.width > side:
+        image = image.resize((side, round(image.height * side / image.width)))
+    canvas = ImageDraw.Draw(image)
+    try:
+        font = ImageFont.load_default(size=max(10, image.width // 36))
+    except TypeError:
+        font = ImageFont.load_default()
+    W, H = image.size
+    for mark in marks:
+        x, y, w, h = mark.rect
+        box = (int(x * W), int(y * H), int((x + w) * W), int((y + h) * H))
+        colour = (255, 40, 120)
+        canvas.rectangle(box, outline=colour, width=2)
+        text = mark.id
+        tw, th = canvas.textbbox((0, 0), text, font=font)[2:]
+        tag = (box[0], max(0, box[1] - th - 2), box[0] + tw + 4, max(th + 2, box[1]))
+        canvas.rectangle(tag, fill=colour)
+        canvas.text((tag[0] + 2, tag[1]), text, fill=(255, 255, 255), font=font)
+    out = io.BytesIO()
+    image.save(out, "JPEG", quality=70)
+    return out.getvalue()

mobile_agent/mcp_server/__init__.py

@@ -0,0 +1,3 @@
+"""`mobster mcp`: the MCP server that coding agents use to check iOS screens."""
+
+__all__ = ["cli", "protocol", "session", "tools"]

mobile_agent/mcp_server/__main__.py

@@ -0,0 +1,29 @@
+"""`python -m mobile_agent.mcp_server`: the same server as `mobster mcp`, with the same flags."""
+
+import argparse
+import sys
+
+DESCRIPTION = (
+    "Serve Mobster's tools to a coding agent over MCP (stdio). The agent starts a check\n"
+    "with verify_start, drives the app on a headless simulator, and gets the verdict\n"
+    "from verify_finish. With an OpenAI or Anthropic key, smart runs the steps itself\n"
+    "(Smart), on the model Smart picks for that key or the one MOBSTER_SMART_MODEL names.")
+
+
+def build_parser():
+    from pathlib import Path
+
+    from .cli import add_arguments
+    parser = argparse.ArgumentParser(prog="python -m mobile_agent.mcp_server", description=DESCRIPTION,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False)
+    add_arguments(parser, {"path_type": lambda value: Path(value).expanduser()})
+    return parser
+
+
+def main(argv=None):
+    from .cli import run
+    return run(build_parser().parse_args(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mobile_agent/mcp_server/action_journal.py

@@ -0,0 +1,24 @@
+"""Durable intent and outcome for direct MCP mutations, including key-less checks."""
+
+from contextlib import contextmanager
+from pathlib import Path
+from uuid import uuid4
+
+from ..journal import append_jsonl
+
+
+@contextmanager
+def record_action(run_dir, operation):
+    # Arguments can contain passwords, codes and private URLs. Record only the
+    # operation here; the existing step records own their redacted descriptions.
+    path = Path(run_dir) / "actions.jsonl"
+    action = {"id": uuid4().hex, "operation": operation}
+    append_jsonl(path, {"event": "action_started", **action})
+    try:
+        yield
+    except BaseException:
+        # A timeout or failed screen read cannot prove that nothing happened.
+        append_jsonl(path, {"event": "action_finished", **action, "outcome": "unknown"})
+        raise
+    else:
+        append_jsonl(path, {"event": "action_finished", **action, "outcome": "acknowledged"})