mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/server.py

@@ -0,0 +1,363 @@
+"""Loopback-only dashboard API with SSE, cancellation, and one device owner at a time."""
+
+import threading
+import time
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+
+from . import engine_health, engines, eventbus, harness_api, ownership, plan_gate, tracks
+from .api_devices import DeviceRoutesMixin
+from .api_get import GetRoutesMixin
+from .api_http import ApiBase
+from .api_post import WriteRoutesMixin
+from .app_icons import AppIcons
+from .cli_constants import DEFAULT_WDA_URL
+from .costs import NANODOLLARS
+from .device_info import DeviceInfo
+from .device_manager import DeviceManager
+from .fanout import GROUP_ID, Fanout
+from .fleet import Fleet
+from .installed_apps import InstalledApps, InventoryCache
+from .journal import Journal, JournalError
+from .manual_control import ManualControl
+from .paths import user_data_dir
+from .pool import default_mjpeg_url
+from .power_watch import PowerWatch
+from .run import Run
+from .runtime_create import TaskStartMixin
+from .runtime_driver import PhoneDriverMixin
+from .runtime_limits import LimitsMixin
+from .runtime_queue import PhoneQueueMixin
+from .runtime_runs import RunActionsMixin
+from .runtime_status import StatusMixin
+from .runtime_teach import LiveViewMixin
+from .runtime_work import TaskWorkMixin
+from .setup_service import SetupService
+from .wda.frame_store import FrameStore
+from .wda.video import WdaVideo
+from .workflows.service import Workflows
+
+DONE_ACTION_EVENTS = frozenset({"action_acknowledged", "frontier_action"})
+ACTING_EVENTS = frozenset({"action_started", "app_launch_started", "frontier_action", "frontier_failed",
+                           "skill_started",
+                           "frontier_app_relaunched", "frontier_cleanup"})
+IDLE_OPERATIONS = frozenset({None, "DONE", "BLOCKED", "WAIT"})
+
+
+def may_have_acted(event):
+    # Fails closed: a decision is journaled before it is sent, and one whose ops can't be read counts as acting.
+    kind = event.get("event")
+    if kind in ACTING_EVENTS:
+        return True
+    if kind != "frontier_decision":
+        return False
+    ops = event.get("ops")
+    return not isinstance(ops, list) or any(op not in IDLE_OPERATIONS for op in ops)
+
+
+class Runtime(PhoneQueueMixin, PhoneDriverMixin, LiveViewMixin, LimitsMixin, StatusMixin, TaskStartMixin, TaskWorkMixin,
+              RunActionsMixin):
+    # `mobster run` and the terminal UI: a task that names no device runs on the one target, without the device list.
+    one_target = False
+    plan_gate = None
+    # The demo (a scripted phone and model) never acts on a phone, so it needs no plan.
+    needs_plan = True
+    # The sentence a yes to an outside agent's approval is refused with, or None: the hosted review demo
+    # (remote/review.py) sets it, and Run.answer_approval then takes no yes from the Mac window, a link or the console.
+    approvals_closed = None
+    # The device list's class (fleet.Fleet); a scripted world of several phones gives its own (tests/fleet_world.py).
+    fleet_class = Fleet
+
+    def __init__(self, config):
+        # The tracks register first: their schemas migrate when the journal opens below.
+        tracks.load()
+        self.config = config
+        self.runs = {}
+        self.lock = threading.Lock()
+        self._conversation_apps = {}
+        self.listeners = harness_api.ListenerQueue()
+        self.bus = eventbus.bus
+        # run id -> device id: one live run per device.
+        self.active_runs = {}
+        self.queues = ownership.Queues(ttl=ownership.ttl_seconds())
+        self.queued = {}
+        self.queue_timers = {}
+        self.pool = self.make_pool(config)
+        self.inventory = {}
+        self.inventory_at = 0
+        self.inventory_lock = threading.Lock()
+        self.inventory_ok = False
+        self.wda_health_at, self.wda_ready = 0, False
+        # Re-resolved from WDA itself: a runner restart replaces the session, so a fixed --session would fail.
+        self.wda_session_id, self.wda_session_lock = getattr(config, "session", None), threading.Lock()
+        self.closing = False
+        self.threads = set()
+        self.streams = threading.BoundedSemaphore(8)
+        # Set by ``serve`` once it listens; Mobster Remote's bridge waits on ``serving``.
+        self.local_api = None
+        self.serving = threading.Event()
+        self.device_info = self.make_device_info(config)
+        self.awake = self.make_awake_lease()
+        self.power = PowerWatch(self)
+        # WDA's MJPEG port is paired with its own (8100 -> 9100, 8203 -> 9203), never another device's stream.
+        self.video = (WdaVideo(config.wda_url, mjpeg_url=default_mjpeg_url(config.wda_url),
+                               session=self.current_wda_session,
+                               on_activity=self.video_activity, quality=self.video_quality())
+                      if getattr(config, "wda_url", None) else self.make_video(config))
+        self.control = ManualControl(config.wda_url, self.wda_session) if getattr(config, "wda_url", None) else None
+        self.controlled_at = {}
+        self.lessons, self.lessons_lock = {}, threading.Lock()
+        self.journal = Journal(getattr(config, "state_db", None))
+        state_db = getattr(config, "state_db", None)
+        self.frames = FrameStore(Path(state_db).parent / "frames" if state_db else None)
+        self.model_reach = engines.ModelReach()
+        self.scheduler = None
+        self.manager = self.setup = None
+        self.installed_apps = None
+        data_dir = Path(getattr(config, "data_dir", None) or user_data_dir())
+        self.plan_gate = plan_gate.for_runtime(config, data_dir) if self.needs_plan else None
+        self.icons = AppIcons(data_dir / "app-icons", allowed=self.icon_bundles)
+        if getattr(config, "manage_device", False):
+            env_file = getattr(config, "env_file", None) or data_dir / "agent.env"
+            self.manager = DeviceManager(data_dir, config.wda_url or DEFAULT_WDA_URL)
+            self.setup = SetupService(self, self.manager, env_file)
+            self.device_info = DeviceInfo(manager=self.manager)
+            threading.Thread(target=self.manager.watch, args=(self.manager.watching,), name="mobster-device-watch",
+                             daemon=True).start()
+            # --socket exists only with the optional extension, which the signed app doesn't ship.
+            if not getattr(config, "socket", None):
+                self.installed_apps = InstalledApps(self.manager.device,
+                                                    cache=InventoryCache(data_dir / "installed-apps"),
+                                                    over_wifi=self.manager.wifi_udid)
+        self.fleet = self.fleet_class(self)
+        self.engine_health = engine_health.EngineHealth(publish=self.publish, boot=self.bus.boot)
+        self.engine_sampler = engine_health.Sampler(self)
+        try:
+            self.workflows = Workflows(self)
+            for item in self.journal.load():
+                run = Run(item["app"], item["goal"], item["mode"], id=item["id"],
+                    status=item["status"], created_at=item["createdAt"] / 1000,
+                    finished_at=item["finishedAt"] / 1000 if item["finishedAt"] is not None else None,
+                    events=item["events"], summary=item["summary"],
+                    idempotency_key=item.get("idempotencyKey"), output_schema=item.get("outputSchema"),
+                    output_format=item.get("outputFormat", "json"),
+                    allowed_bundles=(frozenset(item["allowedBundles"]) if item.get("allowedBundles") else None),
+                    journal=self.journal, engine=item.get("engine"), estimate_usd=item.get("estimateUsd"),
+                    reviewed_ok=item.get("reviewedOk") is True, frames=self.frames,
+                    device_id=item.get("device"), device_name=item.get("deviceName"),
+                    group_id=item["groupId"] if isinstance(item.get("groupId"), str)
+                    and GROUP_ID.fullmatch(item["groupId"]) else None,
+                    extras=dict(item.get("extras") or {}),
+                    origin=item.get("origin") if item.get("origin") in harness_api.ORIGINS else "api",
+                    pinned_apps=item.get("pinnedApps") if isinstance(item.get("pinnedApps"), list) else None,
+                    block_state=dict(item["blockState"]) if isinstance(item.get("blockState"), dict) else {},
+                    shares=[dict(x) for x in item["shares"] if isinstance(x, dict)][:50]
+                    if isinstance(item.get("shares"), list) else [],
+                    issues=[dict(x) for x in item["issues"] if isinstance(x, dict)][:50]
+                    if isinstance(item.get("issues"), list) else [])
+                if isinstance(item.get("costUsd"), (int, float)):
+                    run.cost_nanodollars, run.cost_known = round(item["costUsd"] * NANODOLLARS), True
+                self.runs[run.id] = run
+                if run.finished_at is None:
+                    # A task that waited for a held phone never started a worker: it acted on nothing.
+                    waited = not any(e.get("event") == "run_started" or may_have_acted(e) for e in run.events)
+                    run.finish({"event": "result", "status": "interrupted", "independently_verified": False,
+                        "reason": ownership.QUEUE_RESTART if waited else
+                        "The service restarted. An in-flight action may have run; this task was not resumed.",
+                        "actions": sum(e["event"] in DONE_ACTION_EVENTS for e in run.events),
+                        "actions_may_have_run": any(may_have_acted(e) for e in run.events)})
+                    if run.summary.get("persistence_failed"):
+                        raise JournalError("Interrupted task recovery could not be saved")
+        except Exception:
+            self.journal.close()
+            raise
+        self.fanout = Fanout(self)
+        self.start_services()
+
+    def start_services(self):
+        self._services = list(harness_api.services())
+        for start, _close in self._services:
+            try:
+                start(self)
+            except Exception:  # noqa: BLE001 -- a track's service never stops Mobster from starting
+                harness_api.log.exception("a service failed to start")
+
+    def close_services(self):
+        for _start, close in reversed(getattr(self, "_services", ())):
+            try:
+                close(self)
+            except Exception:  # noqa: BLE001
+                harness_api.log.exception("a service failed to close")
+        self._services = []
+
+    def scripted_models(self):
+        return type(self).smart_client is not Runtime.smart_client
+
+    def device_busy(self, device_id):
+        with self.lock:
+            return any(value == device_id for value in self.active_runs.values()) or any(
+                getattr(run, "device_id", None) == device_id and run.id in self.active_runs
+                for run in self.runs.values())
+
+    def run_context(self, run):
+        phone = self.phone_for(run) if getattr(self, "fleet", None) is not None else None
+        kind = getattr(phone, "kind", None) if phone is not None else None
+        if kind is None and getattr(self.config, "wda_url", None):
+            kind = "usb" if getattr(self, "manager", None) is not None else "wda"
+        state_db = getattr(self.config, "state_db", None)
+        return harness_api.RunContext(
+            run_id=run.id, goal=run.goal, engine=run.engine or engines.SMART, origin=run.origin,
+            app_bundle=run.app.get("bundleId"), device_id=run.device_id, device_kind=kind,
+            extras=harness_api.frozen_mapping(run.extras), data_dir=Path(state_db).parent if state_db else None,
+            emit=run.emit, clarify=harness_api.clarify_only(run.request_approval), cancelled=run.stop.is_set,
+            runtime=self, account_context=run.account_context, workflow_id=run.workflow_id)
+
+    def current_wda_session(self):
+        # Without a session the stream stays at full size and framerate (measured: 1178x2556 at ~9.5 fps on an
+        # iPhone 15 Pro). WDA keeps MJPEG settings server-wide, so making one here changes nothing a run relies on.
+        try:
+            return self.wda_session()
+        except Exception:
+            return None
+
+    def video_activity(self, active):
+        self.awake.acquire("video") if active else self.awake.release("video")
+
+    @property
+    def active(self):
+        return next(iter(self.active_runs), None)
+
+    def capacity(self):
+        if self.pool is None:
+            fleet = getattr(self, "fleet", None)
+            if fleet is None:
+                return 0 if self.active_runs else 1
+            return 0 if self.run_on(fleet.primary) else 1
+        free = self.pool.position()
+        return max(0, free["healthy"] - free["busy"])
+
+    def publish(self, topic, event):
+        try:
+            (getattr(self, "bus", None) or eventbus.bus).publish(topic, event)
+        except Exception:  # noqa: BLE001
+            harness_api.log.exception("event bus publish failed")
+
+    def close(self, timeout=25):
+        deadline = time.monotonic() + timeout
+        self.engine_sampler.close()
+        # Stop every task before the runners and relays go: quitting the Mac app gives this process only SIDECAR_GRACE
+        # (desktop/src-tauri/src/sidecar.rs).
+        with self.lock:
+            self.closing = True
+            waiting = self.drop_queues()
+            for run in self.runs.values():
+                if run.finished_at is None:
+                    with run.condition:
+                        if not run.stop.is_set():
+                            run.quitting = True
+                        run.stop.set()
+                        run.condition.notify_all()
+        scheduled_checks = getattr(self, "test_schedules", None)
+        if scheduled_checks is not None and not scheduled_checks.close(max(0.0, deadline - time.monotonic())):
+            return False  # Its suite may still dispatch: keep the phone and journal owned.
+        from .projects import service as project_service
+        projects = project_service.service_for(self)
+        if projects is not None:
+            if projects.begin_close() is False:
+                return False
+            if not projects.join(max(0.0, deadline - time.monotonic())):
+                return False  # An accepted project check may still dispatch or write its report.
+        for timer in list(self.queue_timers.values()):
+            timer.cancel()
+        for run in waiting:
+            try:
+                self.finished(run)
+            except Exception:  # noqa: BLE001 -- closing goes on
+                harness_api.log.exception("run_finished listeners failed")
+        if not getattr(self.config, "keep_runner", False):
+            phones = list(getattr(getattr(self, "fleet", None), "phones", {}).values())
+            for manager in [self.manager] + [getattr(phone, "manager", None) for phone in phones]:
+                begin = getattr(manager, "begin_close", None)
+                if callable(begin):
+                    try:
+                        begin()
+                    except Exception:  # noqa: BLE001 -- closing goes on; close() below stops it the slow way
+                        pass
+        workflows = getattr(self, "workflows", None)
+        if workflows is not None and hasattr(workflows, "close"):
+            if workflows.close() is False:
+                return False  # a batch may still write its ending to the journal
+        if self.scheduler and not self.scheduler.close(max(0.0, deadline - time.monotonic())):
+            return False  # a scheduler dispatch may still be in flight
+        self.video.close()
+        if getattr(self, "control", None) is not None:
+            self.control.close()
+        if getattr(self, "fleet", None) is not None:
+            self.fleet.close(keep_runner=getattr(self.config, "keep_runner", False))
+        if not getattr(self.config, "keep_runner", False):
+            from .sim import wda as sim_wda
+            sim_wda.stop_started()  # a simulator's runner this process started goes with it
+        if self.manager is not None:
+            if getattr(self.config, "keep_runner", False):
+                # A development reload: the runner and relay outlive this process and the next server adopts them.
+                self.manager.watching.set()
+            else:
+                self.manager.close()
+        with self.lock:
+            self.closing = True
+            threads = list(self.threads)
+            for run in self.runs.values():
+                if run.finished_at is None:
+                    run.stop.set()
+        self.awake.close()
+        self.power.close()
+        for thread in threads:
+            thread.join(max(0, deadline - time.monotonic()))
+        if any(thread.is_alive() for thread in threads):
+            # Keep the leases and journal until exit: an old worker might still dispatch.
+            return False
+        if getattr(self, "fanout", None) is not None:
+            self.fanout.close()
+        if getattr(self, "listeners", None) is not None:
+            if not self.listeners.close(timeout=max(0.0, deadline - time.monotonic())):
+                return False
+        self.close_services()
+        if getattr(self, "frames", None) is not None:
+            self.frames.close()
+        self.journal.close()
+        return True
+
+
+class ApiHandler(GetRoutesMixin, WriteRoutesMixin, DeviceRoutesMixin, ApiBase):
+    pass
+
+
+def make_handler(runtime, token=None, app_session=None, setup_token=None):
+    # token None (tests only) leaves requests unauthenticated.
+    return type("Handler", (ApiHandler,), {"runtime": runtime, "token": token, "app_session": app_session,
+                                           "setup_token": setup_token})
+
+
+class BoundedServer(ThreadingHTTPServer):
+    daemon_threads = True
+    request_queue_size = 32
+
+    def __init__(self, *args, **kwargs):
+        self.capacity = threading.BoundedSemaphore(32)
+        super().__init__(*args, **kwargs)
+
+    def process_request(self, request, address):
+        if not self.capacity.acquire(blocking=False):
+            self.shutdown_request(request)
+            return
+        try:
+            super().process_request(request, address)
+        except Exception:
+            self.capacity.release()
+            raise
+
+    def process_request_thread(self, request, address):
+        try:
+            super().process_request_thread(request, address)
+        finally:
+            self.capacity.release()

mobile_agent/service_support.py

@@ -0,0 +1,42 @@
+"""What the per-runtime services (connectors, outlets, plugins) share: their registry, ``ms`` and the app-only check."""
+
+import threading
+
+from .api_errors import APIError
+
+APP_ONLY = "Do this in the Mobster app."
+
+
+class ServiceRegistry:
+    """One service per Runtime, keyed by the runtime object."""
+
+    def __init__(self):
+        self._items = {}
+        self._lock = threading.Lock()
+
+    def get(self, runtime):
+        with self._lock:
+            return self._items.get(id(runtime))
+
+    def add(self, runtime, service):
+        with self._lock:
+            self._items[id(runtime)] = service
+
+    def pop(self, runtime):
+        with self._lock:
+            return self._items.pop(id(runtime), None)
+
+    def all(self):
+        with self._lock:
+            return list(self._items.values())
+
+
+def ms(seconds):
+    return round(seconds * 1000) if isinstance(seconds, (int, float)) else None
+
+
+def needs_app(request, message=APP_ONLY):
+    """403 app_only when this agent has a Mac app session and the request lacks it."""
+    local = getattr(request.runtime, "local_api", None) or {}
+    if local.get("app_session") and not request.app_session:
+        raise APIError(message, 403, "app_only")

mobile_agent/settle/__init__.py

@@ -0,0 +1 @@
+"""Predictive settle and screen streams (MOBSTER_EXPERIMENTAL_SETTLE). See predict.py and docs/waiting.mdx."""

mobile_agent/settle/predict.py

@@ -0,0 +1,102 @@
+"""Predictive settle: wait the learned p90, read once, let the next verify read prove it.
+
+Polling is the floor: the settle loop reads until two reads agree for 0.3 to 0.5 s. When an app's transition has
+settled in X ms at least nine times in ten, a step can sleep X ms, take one read and go on. The next observation (the
+frontier's verify read, taken while the model thinks) proves the read, the same contract as ``single_read``; so it
+runs only for callers that verify (``single_read`` or ``early``), and only with ``MOBSTER_EXPERIMENTAL_SETTLE``.
+
+Nothing here decides a step. A prediction that is missing, a read that still shows the screen before the action, or
+pixels that moved during the read fall back to the polling loop, unchanged. Each settle row records what was
+predicted and what happened (``predicted``: ms, ``outcome``: hit, miss reason or none; ``arrival_ms``).
+"""
+
+import threading
+import time
+
+from .store import SettleStore, enabled, shape_of
+from .stream import WdaMjpegObserver
+
+_store = None
+_store_lock = threading.Lock()
+STABLE_SECONDS = .18  # frame_clock.STABLE_SECONDS: the stream must be still this long before the read ended
+READ_MARGIN_SECONDS = .35  # wda.observe.WDA_READ_MARGIN_SECONDS
+
+
+def default_store():
+    global _store
+    with _store_lock:
+        if _store is None:
+            _store = SettleStore()
+        return _store
+
+
+def reset_default_store():
+    global _store
+    with _store_lock:
+        _store = None
+
+
+def arrival_ms(started, reads, before):
+    """When the final screen first showed, in ms after ``started``: the first read of the trailing run of agreeing
+    reads, or None when the run is shorter than two reads or shows the screen before the action."""
+    if len(reads) < 2 or reads[-1][2] == before:
+        return None
+    first = len(reads) - 1
+    while first > 0 and reads[first - 1][2] == reads[-1][2]:
+        first -= 1
+    if len(reads) - first < 2:
+        return None
+    return max(0.0, (reads[first][0] - started) * 1000)
+
+
+def predicted_settle(driver, snapshot, action, watch, clock, deadline, started, cap, reads, info,
+                     store=None):
+    """A changed read after the learned wait, or None (``info`` says why). Reads happen at most once."""
+    if not enabled() or not action:
+        return None
+    store = store or default_store()
+    app, shape = snapshot.bundle_id, shape_of(action.get("op"))
+    wait = store.p90(app, shape)
+    if wait is None:
+        info["outcome"] = "none"
+        return None
+    info["predicted"], info["shape"] = round(wait, 1), shape
+    # The wait is counted from the dispatch's end, which is when ``started`` was taken.
+    pause = min(started + wait / 1000, cap - READ_MARGIN_SECONDS) - time.monotonic()
+    if pause > 0:
+        time.sleep(pause)
+    read_started = time.monotonic()
+    if deadline.end - read_started <= READ_MARGIN_SECONDS:
+        info["outcome"] = "no_time"
+        return None
+    try:
+        current = driver.observe(timeout=deadline.remaining())
+    except Exception:  # TransportError or TimeoutError: the polling loop owns the failure.
+        info["outcome"] = "read_failed"
+        return None
+    now = time.monotonic()
+    reads.append((read_started, now, current.content_fingerprint))
+    if current.content_fingerprint == snapshot.content_fingerprint:
+        info["outcome"] = "unchanged"
+        return None
+    if WdaMjpegObserver(clock, watch).still_between(read_started - STABLE_SECONDS, now) is False:
+        info["outcome"] = "moved"
+        return None
+    info["outcome"] = "hit"
+    return current
+
+
+def annotate(driver, entry, action, snapshot, result, started, reads, info):
+    """Put the prediction on the settle row, remember it on the driver (``predicted_settle``) and learn from the
+    settle. Learns only from settles that ended on agreeing reads; a hit is one read, so it teaches nothing."""
+    if not enabled():
+        return
+    arrival = arrival_ms(started, reads, snapshot.content_fingerprint)
+    shape = shape_of((action or {}).get("op"))
+    if arrival is not None:
+        entry["arrival_ms"] = round(arrival, 1)
+        default_store().add(snapshot.bundle_id, shape, arrival)
+    if info is not None and "outcome" in info:
+        entry.update({key: info[key] for key in ("predicted", "outcome", "shape") if key in info})
+    driver.predicted_settle = {key: entry.get(key) for key in ("shape", "predicted", "outcome", "arrival_ms")
+                               if entry.get(key) is not None} or None

mobile_agent/settle/replay.py

@@ -0,0 +1,92 @@
+"""Offline replay of recorded settles: what predictive settle would have done on frameclock.jsonl logs.
+
+    python -m mobile_agent.settle.replay LOG [LOG ...]
+
+Reads ``settle`` rows (``MOBSTER_FRAME_CLOCK_LOG`` files) in time order, learns each key's arrival the way a run does,
+and for every settle that polled (path ax, frame or frame_fallback) asks whether the learned p90 would have been
+enough. A row records no arrival unless it was logged with the switch on (``arrival_ms``), so the replay estimates it:
+``stable_ms`` when the row has it, else ``settle_ms`` minus the polling loop's quiet period (0.5 s on AX alone, 0.3 s
+with the pixels). The wall time of a hit is the wait plus one read; of a miss, the wait, one read and then the
+whole polled settle (an upper bound). These are projections from recordings, not a measured effect of the switch.
+"""
+
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+from .store import SettleStore, shape_of
+
+POLLED = {"ax", "frame", "frame_fallback"}
+QUIET_AX_MS, QUIET_PIXELS_MS, READ_MS = 500.0, 300.0, 200.0
+
+
+def estimate_arrival(row):
+    if row.get("arrival_ms") is not None:
+        return float(row["arrival_ms"])
+    if row.get("stable_ms") is not None:
+        return float(row["stable_ms"])
+    quiet = QUIET_PIXELS_MS if row.get("path") == "frame" else QUIET_AX_MS
+    return max(0.0, float(row["settle_ms"]) - quiet)
+
+
+def load(paths):
+    rows = []
+    for path in paths:
+        with open(path, encoding="utf-8") as handle:
+            for line in handle:
+                try:
+                    row = json.loads(line)
+                except ValueError:
+                    continue
+                if row.get("kind") == "settle" and row.get("changed") and "settle_ms" in row:
+                    rows.append(row)
+    return sorted(rows, key=lambda row: row.get("at", 0))
+
+
+def replay(rows):
+    with tempfile.TemporaryDirectory() as folder:
+        store = SettleStore(Path(folder) / "durations.json")
+        totals = {"settles": 0, "predicted": 0, "hits": 0, "misses": 0, "polled_ms": 0.0, "with_ms": 0.0,
+                  "polled_reads": 0, "with_reads": 0, "no_prediction": 0}
+        for row in rows:
+            if row.get("path") not in POLLED:
+                continue
+            app, shape = row.get("bundle") or "", shape_of(row.get("op"))
+            arrival, wall, reads = estimate_arrival(row), float(row["settle_ms"]), int(row.get("reads", 1))
+            totals["settles"] += 1
+            totals["polled_ms"] += wall
+            totals["polled_reads"] += reads
+            wait = store.p90(app, shape)
+            if wait is None:
+                totals["no_prediction"] += 1
+                totals["with_ms"], totals["with_reads"] = totals["with_ms"] + wall, totals["with_reads"] + reads
+                store.add(app, shape, arrival)
+            elif wait >= arrival:
+                totals["predicted"] += 1
+                totals["hits"] += 1
+                totals["with_ms"] += wait + READ_MS
+                totals["with_reads"] += 1
+            else:
+                totals["predicted"] += 1
+                totals["misses"] += 1
+                totals["with_ms"] += wait + READ_MS + wall
+                totals["with_reads"] += 1 + reads
+                store.add(app, shape, arrival)
+    return totals
+
+
+def report(totals):
+    n = totals["settles"] or 1
+    lines = [f"{totals['settles']} polled settles; {totals['no_prediction']} had no prediction yet, "
+             f"{totals['hits']} hits, {totals['misses']} misses ({totals['predicted']} predicted)",
+             f"step settle wall: {totals['polled_ms'] / n:.0f} ms polled -> {totals['with_ms'] / n:.0f} ms predicted "
+             f"(mean per settle)",
+             f"AX reads per settle: {totals['polled_reads'] / n:.2f} -> {totals['with_reads'] / n:.2f}"]
+    if totals["predicted"]:
+        lines.append(f"hit rate when predicted: {totals['hits'] / totals['predicted']:.0%}")
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    print(report(replay(load(sys.argv[1:]))))