mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/frontier/stages/guards.py

@@ -0,0 +1,132 @@
+"""The stages that refuse an action for what came before it: an unknown outcome, a loop, a request that did not ask
+for it."""
+
+import time
+
+from ...task_policy import is_keypad_key
+from ..act import DISMISS_POINTS, SUBMIT_WINDOW_SECONDS
+from ..protocol import HOUSEKEEPING_OPS, TYPED_OPS
+from ..safety import LOOP_REPEATS, REPEAT_ACT_AGAIN, REPEAT_ACT_FEEDBACK, guard
+
+
+def unknown_outcome(agent, attempt, turn):
+    index = turn.index
+    op, app = attempt.op, attempt.app
+    effect = agent._effect(op, attempt.element, app, attempt.text, agent._app_name(turn.snapshot), turn.snapshot)
+    attempt.unknown_repeat = None  # "retry" or "ask": this action repeats one whose outcome was unknown
+    attempt.held = agent._held(effect, turn.snapshot)
+    if attempt.held is not None:
+        # An action that may already have run is never sent again blind: a fresh read settles it first.
+        what = attempt.element.label[:60] if attempt.element else (app or "")
+        attempt.unknown_repeat, fresh, found = agent._recheck_unknown(op, attempt.element, attempt.held, what)
+        if attempt.unknown_repeat == "refuse":
+            turn.feedback = fresh
+            turn.history.append(f"{turn.step}.{index}: {op} {what!r} refused (its earlier outcome was unknown)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "outcome_unknown"})
+            return "stop"
+        turn.snapshot, attempt.element = fresh, found
+
+
+def loops(agent, attempt, turn):
+    index = turn.index
+    op, text, app, element = attempt.op, attempt.text, attempt.app, attempt.element
+    attempt.key = (op, element.label if element else app, text)
+    # A loop is the same control, text and screen: another value or another list row is not one.
+    attempt.spot = (op, (element.locator or element.label) if element else app,
+            text if op in TYPED_OPS or op == "SCROLL_TO" else None, turn.snapshot.content_fingerprint)
+    keypad = element is not None and is_keypad_key(op, element.role, element.label)
+    if (not op.startswith("SWIPE_") and op != "DISMISS" and not keypad
+            and turn.recent.count(attempt.spot) >= LOOP_REPEATS - 1):
+        turn.feedback = (f"Refused: this is the {LOOP_REPEATS}rd time recently you {op} "
+                    f"{(element.label if element else app)!r}; you are going in circles. Use a different "
+                    "route: the app's search, another tab or menu, scrolling, or skip this part and move "
+                    "on to the next part of the request.")
+        turn.history.append(f"{turn.step}.{index}: {op} {(element.label[:60] if element else app)!r} refused (loop)")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "loop"})
+        return "stop"
+    if op == "DISMISS" and agent.run_state.dismissed == (turn.snapshot.content_fingerprint, len(DISMISS_POINTS) - 1):
+        turn.feedback = ("Refused: tapping outside was tried everywhere and closed nothing. Use the overlay's "
+                    "own controls (Close, Cancel, Done, Back) or choose an item in it.")
+        turn.history.append(f"{turn.step}.{index}: DISMISS refused (tried every point)")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "dismiss_exhausted"})
+        return "stop"
+    # A repeat is the same action on the screen it left (typing) or was made on (a tap).
+    turn.repeats = (
+        turn.repeats + 1
+        if turn.last_done and turn.last_done[0] == attempt.key
+        and turn.snapshot.content_fingerprint == (turn.last_done[2] if op in TYPED_OPS else turn.last_done[1])
+        else 0)
+    if op not in HOUSEKEEPING_OPS and (op in TYPED_OPS and turn.repeats >= 1 or turn.repeats >= 2):
+        turn.feedback = ("Refused: you just did exactly this action. Its effect is already on screen (for "
+                    "typed text, see the field's value). Take the next step instead, e.g. save or confirm.")
+        turn.history.append(f"{turn.step}.{index}: repeat of the previous action refused")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "repeat"})
+        return "stop"
+    act_key = agent._act_key(op, element, turn.snapshot)
+    if act_key is not None and act_key in agent.run_state.acts_done:
+        before_refusal = agent.run_state.act_refused.get(act_key)
+        known = before_refusal if before_refusal is not None else {act_key[2], agent.run_state.act_left[act_key]}
+        new_look = bool(agent.run_state.act_seen[act_key] - known)
+    else:
+        new_look = True
+    if not new_look:
+        # Fails closed: the repeat waits until the run has looked somewhere the act could show.
+        template = REPEAT_ACT_FEEDBACK if act_key not in agent.run_state.act_refused else REPEAT_ACT_AGAIN
+        agent.run_state.act_refused[act_key] = frozenset(agent.run_state.act_seen[act_key] | {act_key[2]})
+        turn.feedback = template.format(label=element.label[:60], step=agent.run_state.acts_done[act_key])
+        turn.history.append(f"{turn.step}.{index}: TAP {element.label[:60]!r} refused (done before on this screen)")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "repeat_act"})
+        return "stop"
+
+
+def requested(agent, attempt, turn):
+    index = turn.index
+    element = attempt.element
+    refused = guard(turn.request, attempt.op, element) if agent._contract is None else None
+    if refused:
+        turn.feedback = refused
+        turn.history.append(f"{turn.step}.{index}: TAP {element.label[:60]!r} refused (not requested)")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": element.label[:120]})
+        return "stop"
+
+
+def refresh(agent, attempt, turn):
+    index = turn.index
+    attempt.refresh_ms = 0
+    if attempt.element is not None and index == 0:
+        # The model took seconds: act on the screen as it is now, or the driver refuses a stale read (splitpay-001).
+        t_refresh = time.monotonic()
+        fresh, reused = agent._refreshed(turn.snapshot)
+        turn.counts["reused"] += reused
+        attempt.refresh_ms = agent._spend("refresh", t_refresh)
+        moved = agent._find(fresh, attempt.element.label, attempt.element.role, attempt.element.locator)
+        if moved is not None:
+            turn.snapshot, attempt.element = fresh, moved
+
+
+def results_still_moving(agent, attempt, turn):
+    index = turn.index
+    op = attempt.op
+    if (op in ("TAP", "LONG_PRESS", "TAP_XY") and agent.run_state.submitted_at is not None
+            and time.monotonic() - agent.run_state.submitted_at < SUBMIT_WINDOW_SECONDS):
+        # Results of a search just submitted may still be moving: tap only on two agreeing reads.
+        t_still = time.monotonic()
+        still, moved_to = agent._still_after_submit(attempt.element)
+        agent._spend("settle", t_still)
+        if still is None:
+            turn.feedback = ("The search results were still changing, so Mobster did not tap. Look at the "
+                        "screen again and choose from what it shows now.")
+            turn.history.append(f"{turn.step}.{index}: {op} held: results still changing after the search")
+            agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                        "reason": "results_moving"})
+            turn.snapshot, turn.unproven = agent._read_once(), False
+            return "stop"
+        turn.snapshot = still
+        if attempt.element is not None:
+            if moved_to is None:
+                agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                            "reason": "target_missing", "wanted": (attempt.element.label or "")[:80]})
+                turn.history.append(f"{turn.step}.{index}: {op} target {attempt.element.label[:60]!r} gone once the "
+                               "results settled; stopped the sequence")
+                return "stop"
+            attempt.element = moved_to

mobile_agent/frontier/stages/points.py

@@ -0,0 +1,155 @@
+"""The stages for the passcode keypad, a skill, and a point or gesture on the screen."""
+
+import dataclasses
+import json
+
+from ...agent_hooks import RESUME_AFTER_SECONDS, GuardVerdict
+from ...task_policy import springboard_keypad
+from ...wda.point_gestures import GESTURE_OPS, validate_gesture_request
+from ..act import LEAVING_OPS
+from ..interrupts import Where, check
+from ..protocol import ELEMENT_OPS
+from ..safety import KEYPAD_REFUSAL, PHONE_LOCKED, SAME_FAILURES_TO_CHECK, GuardStop, declined_result
+
+
+def keypad(agent, attempt, turn):
+    index = turn.index
+    op = attempt.op
+    if op in ELEMENT_OPS or op == "TAP_XY" or op in GESTURE_OPS or op in agent._skill_ops:
+        if springboard_keypad(turn.snapshot):
+            # The lock screen's keypad is never the model's (B2): the guard may unlock or stop.
+            turn.keypad_refusals += 1
+            turn.feedback = KEYPAD_REFUSAL
+            turn.history.append(f"{turn.step}.{index}: {op} refused (passcode screen)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "passcode_keypad"})
+            verdict = agent._guard_check("lock_suspected", turn.history)
+            if verdict is not None and verdict.state == "unlocked":
+                turn.keypad_refusals, turn.snapshot, turn.unproven = 0, agent._observe(), False
+            elif agent.guard is None or turn.keypad_refusals >= SAME_FAILURES_TO_CHECK:
+                raise GuardStop(GuardVerdict("stop", "phone_locked", PHONE_LOCKED + (
+                    "" if turn.actions else " No action was taken.")))
+            return "stop"
+
+
+def skill(agent, attempt, turn):
+    index = turn.index
+    if attempt.op in LEAVING_OPS:
+        agent._reveal_found(turn)
+    op = attempt.op
+    chosen = agent._skill_ops.get(op)
+    if chosen is not None and op == "OPEN_URL" and attempt.act.app and attempt.act.app not in agent.apps:
+        # A web page opens in a listed app only, else in the browser on screen or the default one.
+        attempt.act = dataclasses.replace(attempt.act, app=None)
+    if chosen is not None:
+        if turn.session is not None:
+            turn.session.mark_action()
+        ended, said_back, turn.snapshot, changed, waited = agent._run_skill(
+            chosen, attempt.act, index, turn.aliases, turn.snapshot, turn.request, turn.step, turn.history, turn.after)
+        turn.started += waited
+        agent._waited(waited)
+        turn.unproven = False
+        if ended is None and turn.snapshot is not None:
+            # A link can land in any app: the app in front is checked before anything there is read or done.
+            agent._check_app(turn.snapshot)
+        if ended is not None and "redirect" in ended:
+            instruction = ended["redirect"]
+            turn.request = f"{turn.request}\n\nThe user's later instruction: {instruction}"
+            turn.feedback = f"The user declined {op} and asked instead: {instruction!r}. Do that now."
+            turn.history.append(f"{turn.step}.{index}: {op} declined; user redirected")
+            return "stop"
+        if ended is not None:
+            turn.result = ended
+            return "end"
+        turn.feedback = said_back or ""
+        if changed:
+            turn.actions += 1
+            agent._actions = turn.actions
+            turn.acted = True
+        return "stop"  # a skill's result is looked at before anything else is done
+
+
+def gesture_points(agent, attempt, turn):
+    index = turn.index
+    op = attempt.op
+    if op in GESTURE_OPS:
+        # One gesture, on the screen and picture the model chose its points from, with nothing chained after it.
+        try:
+            attempt.point = validate_gesture_request(GESTURE_OPS[op], json.loads(attempt.text or ""))
+            turn.snapshot = agent.gesture_screen(turn.snapshot, turn.image, attempt.point, turn.request)
+        except (TypeError, ValueError) as error:
+            if check(agent, turn, Where.WAITED) is not None:
+                return "end"
+            turn.feedback = str(error)
+            turn.history.append(f"{turn.step}.{index}: {op} refused")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": op.lower(),
+                        "detail": turn.feedback[:160]})
+            return "stop"
+        del turn.chunk[index + 1:]
+
+
+def point_approval(agent, attempt, turn):
+    index = turn.index
+    op = attempt.op
+    if op == "TAP_XY" or op in GESTURE_OPS:
+        if op == "TAP_XY":
+            if agent._image_mode == "tree" and turn.image is None:
+                attempt.point, why = None, "Read a screenshot before choosing screen points."
+            else:
+                attempt.point, why = agent._tap_xy(turn.request, attempt.act, turn.aliases, turn.snapshot)
+            if why:
+                turn.feedback = why
+                turn.history.append(f"{turn.step}.{index}: TAP_XY {(attempt.text or '')[:40]!r} refused")
+                agent._emit({"event": "frontier_refused", "step": turn.step, "label": "tap_xy",
+                            "detail": why[:160]})
+                return "stop"
+            named = " ".join((attempt.text or "").split())[:40]
+            attempt.app = f"{named} at {attempt.point[0] * 100:.0f},{attempt.point[1] * 100:.0f}"
+        else:
+            named = attempt.app = attempt.point.label
+        attempt.text = None
+        verdict, waited, asked = agent._point_approval(
+            attempt.point.start if op in GESTURE_OPS else attempt.point, turn.snapshot, turn.request, turn.chain_typed,
+            turn.step, turn.session, gesture=attempt.point if op in GESTURE_OPS else None)
+        turn.started += waited  # waiting for the user never spends the task's time
+        agent._waited(waited)
+        if verdict is None and asked and check(agent, turn, Where.WAITED) is not None:
+            return "end"
+        if verdict is None and asked:
+            # The person approved the spot as the screen was: one that moved since is not tapped.
+            if waited >= RESUME_AFTER_SECONDS:
+                agent._guard_check("resume", turn.history)
+            if op in GESTURE_OPS:
+                try:
+                    fresh = agent.gesture_screen(turn.snapshot, turn.image, attempt.point, turn.request)
+                    matched = True
+                except ValueError:
+                    fresh, matched = turn.snapshot, False
+            else:
+                fresh = agent._observe()
+                matched = fresh.content_fingerprint == turn.snapshot.content_fingerprint
+            if check(agent, turn, Where.WAITED) is not None:
+                return "end"
+            if not matched:
+                turn.feedback = "The screen changed while Mobster waited for your OK: look again."
+                turn.history.append(f"{turn.step}.{index}: {op} {named!r} not "
+                                     + ("tapped" if op == "TAP_XY" else "done") + " (the screen changed)")
+                turn.snapshot = fresh
+                return "stop"
+            turn.snapshot = fresh
+        if verdict is not None and verdict.startswith(("refused:", "blocked:")):
+            turn.feedback = (f"Refused: {verdict.split(':', 1)[1]}, or finish with what you found (BLOCKED if "
+                        "the request needs this).")
+            turn.history.append(f"{turn.step}.{index}: TAP_XY {named!r} refused (needs the user's OK)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "needs_approval"})
+            return "stop"
+        if verdict is not None and verdict.startswith("redirected:"):
+            agent.run_state.asked = "redirected"
+            instruction = " ".join(verdict.split(":", 1)[1].split())[:500]
+            turn.request = f"{turn.request}\n\nThe user's later instruction: {instruction}"
+            turn.feedback = (f"The user declined the tap at that spot and asked instead: {instruction!r}. "
+                        "Do that now.")
+            turn.history.append(f"{turn.step}.{index}: TAP_XY {named!r} declined; user redirected")
+            return "stop"
+        if verdict is not None:
+            turn.result = declined_result(verdict)
+            return "end"

mobile_agent/frontier/stages/send.py

@@ -0,0 +1,177 @@
+"""The stages that send an action to the phone and answer its failure."""
+
+import time
+
+from ... import contract_asks as proof_asks
+from ...model_clients import http_detail
+from ...wda.guards import DriverRejection
+from ...wda.point_gestures import GESTURE_OPS
+from ..interrupts import Where, check
+from ..protocol import ELEMENT_OPS, TYPED_OPS
+from ..safety import (
+    CONNECTION_LOST,
+    NOTHING_SENT,
+    SAME_FAILURES_TO_CHECK,
+    SAME_FAILURES_TO_STOP,
+    UNKNOWN_REASON,
+    GuardStop,
+    consequential,
+    failure_feedback,
+    outcome_unknown,
+    stuck_sentence,
+    unknown_advice,
+)
+from ..screen import names_act, reads_back
+
+
+def execute(agent, attempt, turn):
+    index = turn.index
+    op, text, app, point, held, unknown_repeat, gate = (
+        attempt.op, attempt.text, attempt.app, attempt.point, attempt.held, attempt.unknown_repeat, attempt.gate)
+    attempt.before = turn.snapshot
+    attempt.effect = agent._effect(op, attempt.element, app, text, agent._app_name(turn.snapshot), turn.snapshot)
+    attempt.before_unknown = (
+        agent._before_unknown(attempt.element, turn.snapshot) if attempt.effect is not None else None)
+    if unknown_repeat == "retry":
+        # A fresh read showed the earlier attempt did nothing: this is its one more attempt.
+        agent.run_state.effects.settle(held[0], None, effect=held[1], happened=False)
+        agent._emit({"event": "frontier_retry", "step": turn.step, "index": index, "operation": op,
+                    "reason": "outcome_unknown_unchanged"})
+    attempt.t_exec = time.monotonic()
+    agent.run_state.last_receipt = None
+    if turn.session is not None:
+        turn.session.mark_action()
+    if op in GESTURE_OPS:
+        # Journaled before it is sent: after a restart the gesture counts as one that may have run.
+        if check(agent, turn, Where.WAITED) is not None:
+            return "end"
+        agent._emit({"event": "action_started", "operation": op, "target": app})
+        if check(agent, turn, Where.WAITED) is not None:
+            return "end"
+    try:
+        try:
+            attempt.before = agent._execute(op, attempt.element, app, text, turn.snapshot, point)
+        except Exception as error:
+            # Resent only when the request never ran (refused, a dropped session); never a gesture or a point.
+            if op in GESTURE_OPS or op == "TAP_XY" or not agent._recover(error) or not any(
+                    mark in str(error) for mark in NOTHING_SENT):
+                raise
+            turn.snapshot = agent._observe()
+            if attempt.element is not None:
+                attempt.element = agent._find(
+                    turn.snapshot, attempt.element.label, attempt.element.role, attempt.element.locator)
+                if attempt.element is None:
+                    raise
+            attempt.before = agent._execute(op, attempt.element, app, text, turn.snapshot, point)
+        if op in GESTURE_OPS:
+            agent._emit({"event": "action_acknowledged", "step": turn.step, "operation": op})
+        turn.actions += 1
+        agent._actions = turn.actions
+        if attempt.commit_now:
+            agent.run_state.commits += 1
+        if unknown_repeat is not None:
+            agent.run_state.effects.resolve(held[0], None, effect=held[1])
+        turn.acted_since = turn.acted_since or names_act(op, attempt.element, text)
+        if attempt.element is not None:
+            turn.touched.append(attempt.element)
+        if op == "READ_LIST" and agent.run_state.listing:
+            turn.read_rows = list(agent.run_state.listing)
+        agent.run_state.submitted_at = time.monotonic() if op in ("TYPE_SUBMIT", "SUBMIT") else (
+            agent.run_state.submitted_at if op not in ("TAP", "LONG_PRESS", "TAP_XY") else None)
+        if turn.session is not None:
+            # Opening the app already in front is no progress (P1-1).
+            turn.session.on_acted(noop=op == "LAUNCH_APP" and app == getattr(turn.snapshot, "bundle_id", None))
+        if agent._contract is not None:
+            where = agent._app_name(attempt.before)
+            agent._contract.record_action(where, op, getattr(attempt.element, "label", "") or "")
+            if op in TYPED_OPS and text:
+                agent._contract.record_typed(where, attempt.element, text, turn.step)
+                turn.chain_typed.append(text)
+            if gate is not None and gate.item is not None:
+                agent._contract.committed(gate, attempt.element, attempt.before, where, turn.step)
+            if op == "READ_LIST" and agent.run_state.listing:
+                agent._contract.saw_lines(where, turn.step, agent.run_state.listing)
+        turn.acted = turn.acted or op in ELEMENT_OPS or op == "TAP_XY" or op in GESTURE_OPS
+        if agent.run_state.macro_feedback:
+            turn.feedback, agent.run_state.macro_feedback = agent.run_state.macro_feedback, None
+    except GuardStop:
+        raise
+    except Exception as error:
+        attempt.error = error
+
+
+def failure(agent, attempt, turn):
+    error = attempt.error
+    if error is None:
+        return None
+    index = turn.index
+    op, text, app, element, gate, before_unknown, effect = (
+        attempt.op, attempt.text, attempt.app, attempt.element, attempt.gate, attempt.before_unknown, attempt.effect)
+    if op in GESTURE_OPS and not (isinstance(error, DriverRejection) and error.code in error.PRE_DISPATCH):
+        kind = GESTURE_OPS[op]
+        turn.result = {"status": "blocked", "answer": None, "code": "unknown_outcome",
+                        "reason": f"The {kind} outcome is unknown. Inspect the phone before starting another "
+                                  "task; it was not retried."}
+        agent._emit({"event": "frontier_failed", "step": turn.step, "index": index, "operation": op,
+                    "error": f"{kind} outcome unknown", "outcome": "unknown"})
+        return "end"
+    # A request on a dropped session never ran: its outcome is known.
+    lost = agent.run_state.recoveries and any(name in str(error) for name in CONNECTION_LOST if name != "StaleSession")
+    # A tap at a point has no control to read back: any failure past dispatch is held as unknown.
+    unknown = bool(lost) or outcome_unknown(error, op) or (
+        op == "TAP_XY" and not (isinstance(error, DriverRejection) and error.code in error.PRE_DISPATCH))
+    if unknown and effect is not None:
+        agent.run_state.effects.record_outcome(effect[0], None, outcome="unknown", effect=effect[1],
+                                     context=before_unknown)
+    if unknown and gate is not None and gate.item is not None and proof_asks.asks(gate.item):
+        # Any control of this act, on any screen, now asks with the warning, in every mode.
+        agent.run_state.held_items[gate.item.id] = UNKNOWN_REASON
+    turn.feedback = (("The connection to the phone dropped during this action and was restored: it may or "
+                 "may not have happened. " + unknown_advice(op)) if lost
+                else failure_feedback(op, element, text, error))
+    detail = http_detail(error)
+    what = element.label[:60] if element else (app or "")
+    # The line names the target and the status: a bare one repeated a failed launch fifty times (B2).
+    turn.history.append(f"{turn.step}.{index}: {op} {what!r} failed ({detail})")
+    agent._spend("exec", attempt.t_exec)
+    agent._emit({"event": "frontier_failed", "step": turn.step, "index": index, "operation": op,
+                "error": f"{type(error).__name__}: {str(error)[:160]}",
+                "outcome": "unknown" if unknown else "failed",
+                # A picked-up run holds this act too (checkpoints.held_acts).
+                **({"held_act": gate.item.act} if unknown and gate is not None and gate.item is not None
+                   and proof_asks.asks(gate.item) else {})})
+    t_observe = time.monotonic()
+    turn.snapshot, turn.unproven = agent._observe(), False
+    agent._spend("observe", t_observe)
+    if unknown and effect is not None and element is not None:
+        # The look after it: a screen that moved on, or a control that changed, says it happened.
+        if not agent._unchanged(before_unknown, element, turn.snapshot)[0]:
+            agent.run_state.effects.settle(effect[0], None, effect=effect[1], happened=True)
+            turn.feedback += (" The screen changed after it, so Mobster counts it as done and won't send it "
+                         "again.")
+        elif consequential(op, element, turn.snapshot):
+            turn.feedback += " The screen looks unchanged; sending it again needs the user's OK."
+        else:
+            turn.feedback += (" The screen looks unchanged, so it probably didn't happen: if it still looks "
+                         "the same, you may send it once more.")
+    if op in TYPED_OPS and text and element is not None and agent._contract is not None:
+        # Typing that timed out may have landed: a field reading it back holds the agent's text.
+        field = agent._find(turn.snapshot, element.label, element.role, element.locator)
+        if field is not None and field.editable and reads_back(field.value or "", text):
+            agent._contract.record_typed(agent._app_name(turn.snapshot), field, text, turn.step)
+            turn.feedback += " The field now reads back exactly the text you wrote: don't type it again."
+    same = (op, (element.locator or element.label) if element else app, text)
+    turn.failures[same] = turn.failures.get(same, 0) + 1
+    if op == "LAUNCH_APP" and detail == "HTTP 400" or turn.failures[same] >= SAME_FAILURES_TO_CHECK:
+        # iOS refuses to bring an app forward over the lock screen or a passcode sheet.
+        verdict = agent._guard_check("launch_failed" if op == "LAUNCH_APP" else "lock_suspected",
+                                    turn.history)
+        if verdict is not None and verdict.state == "unlocked":
+            turn.failures.clear()
+            turn.snapshot = agent._observe()
+    if turn.failures.get(same, 0) >= SAME_FAILURES_TO_STOP:
+        name = agent.apps.get(app, app) if op == "LAUNCH_APP" else what
+        sentence = stuck_sentence(op, name, detail, turn.actions)
+        turn.result = {"status": "blocked", "answer": None, "reason": sentence}
+        return "end"
+    return "stop"

mobile_agent/frontier/turn.py

@@ -0,0 +1,151 @@
+"""The two records a run keeps: Turn, what the turn loop and its actions share, and RunState, what the run has done."""
+
+import dataclasses
+import threading
+from collections import deque
+from typing import Any
+
+from ..effect_ledger import EffectLedger
+from ..harness_api import InitialState
+from ..state import Element, Snapshot
+from .actions import Action
+from .screen import Provenance, _Verify
+
+
+# The turn loop's locals that the per-action step shares; slots turn a misspelt name into an AttributeError.
+@dataclasses.dataclass(slots=True, kw_only=True)
+class Turn:
+    request: str
+    started: float
+    recent: deque
+    session: Any  # a longform session; the engine does not import longform
+    data_json: bool
+    stopped: dict
+    acted: bool = False
+    acted_since: bool = False
+    actions: int = 0
+    after: str | None = None
+    aliases: dict[str, Element] = dataclasses.field(default_factory=dict)
+    calls: int = 0
+    chain_typed: list[str] = dataclasses.field(default_factory=list)
+    checked_front: bool = False
+    chunk: list[Action] = dataclasses.field(default_factory=list)
+    counts: dict[str, int] = dataclasses.field(
+        default_factory=lambda: {"reused": 0, "verified": 0, "verify_changed": 0})
+    covered: int = 0
+    failures: dict[tuple, int] = dataclasses.field(default_factory=dict)
+    feedback: str = ""
+    finished: bool = False
+    history: list[str] = dataclasses.field(default_factory=list)
+    idle_waits: int = 0
+    image: dict | None = None
+    index: int = 0
+    keypad_refusals: int = 0
+    last_done: tuple | None = None
+    out: dict = dataclasses.field(default_factory=dict)
+    read_rows: list[str] | None = None
+    refusals: dict[str, int] = dataclasses.field(default_factory=dict)
+    repeats: int = 0
+    request_checked: bool = False
+    reserve: float = 0.0
+    result: dict = dataclasses.field(default_factory=lambda: {"status": "max_steps", "answer": None})
+    snapshot: Snapshot | None = None
+    spent: float | None = None
+    step: int = 0
+    touched: list[Element] = dataclasses.field(default_factory=list)
+    turn_rows: set[str] | None = None
+    turn_screen: Snapshot | None = None
+    unproven: bool = False
+    untaken: tuple[str, str] | None = None
+    plan: str = ""
+    notes: list[str] = dataclasses.field(default_factory=list)
+    max_actions: int = 0
+    source_reads: tuple = (None, None)
+    compiled: list = dataclasses.field(default_factory=list)
+    worker: threading.Thread | None = None
+    pending: bool = False
+    unpriced: str = ""
+    unrunnable: int = 0
+    final: str | None = None
+    capped: bool = False
+    operations: tuple[str, ...] = ()
+    lines: list[str] = dataclasses.field(default_factory=list)
+    prep_ms: int = 0
+    messages: list | None = None
+    verify: _Verify | None = None
+    programmed: bool = False
+    proposed: list[Action] = dataclasses.field(default_factory=list)
+    unrun: str | None = None
+    usage: dict = dataclasses.field(default_factory=dict)
+    latency_ms: int = 0
+
+
+# What one run has done so far; run() starts a new one, so nothing carries over from an earlier run.
+@dataclasses.dataclass(slots=True, kw_only=True)
+class RunState:
+    started: float
+    initial: InitialState | None = None
+    held_acts: frozenset = frozenset()
+    notes: list[str] = dataclasses.field(default_factory=list)
+    plan: str = ""
+    compile_worker: threading.Thread | None = None
+    calls: int = 0
+    timing: dict[str, float] = dataclasses.field(default_factory=dict)
+    cost_offset: float = 0.0
+    recovery_spent: float = 0.0
+    stall_spent: float = 0.0
+    unlocks: int = 0
+    secrets: set[str] = dataclasses.field(default_factory=set)
+    secret_rects: list[tuple] = dataclasses.field(default_factory=list)
+    skill_images: list[tuple] = dataclasses.field(default_factory=list)
+    imaged_app: str | None = None
+    blind: int = 0
+    image_why: str = "off"
+    cards: int = 0
+    commits: int = 0
+    held_items: dict[str, str] = dataclasses.field(default_factory=dict)
+    injected: bool = False
+    ask_reason: str | None = None
+    acting_as: str | None = None
+    media_checked: bool | None = None  # a commit_check preview's verdict on the commit's screen; None: none asked
+    approved_send: dict | None = None
+    declined_on: Snapshot | None = None
+    revealed: int | None = None  # the turn whose recorded values were checked for a hidden row (_reveal_found)
+    redirect: tuple[str, str] | None = None
+    part: Any = None  # a learned.parts.PartRun; the engine does not import learned
+    dialog_checked: tuple | None = None
+    battery_pause: bool = False
+    submitted_at: float | None = None
+    dismissed: tuple = (None, 0)
+    receipts: dict = dataclasses.field(default_factory=dict)
+    recoveries: int = 0
+    last_receipt: dict | None = None
+    acts_done: dict = dataclasses.field(default_factory=dict)
+    act_left: dict = dataclasses.field(default_factory=dict)
+    act_seen: dict = dataclasses.field(default_factory=dict)
+    act_refused: dict = dataclasses.field(default_factory=dict)
+    effects: EffectLedger = dataclasses.field(default_factory=EffectLedger)
+    timed_out_reads: int = 0
+    relaunches: int = 0
+    read_bundle: str = ""
+    start_resets: dict = dataclasses.field(default_factory=dict)
+    fresh_launch: tuple | None = None
+    listing: list | None = None
+    macro_feedback: str | None = None
+    list_read: tuple | None = None
+    passed: list[str] = dataclasses.field(default_factory=list)
+    turn_extra: str = ""
+    steered: list = dataclasses.field(default_factory=list)
+    asked: str | None = None
+    turn_log: list[str] = dataclasses.field(default_factory=list)
+    turn_notes: dict[int, list[str]] = dataclasses.field(default_factory=dict)
+    turn_marks: dict[int, list[str]] = dataclasses.field(default_factory=dict)
+    turn_plans: dict[int, str] = dataclasses.field(default_factory=dict)
+    turn_reads: dict[int, str] = dataclasses.field(default_factory=dict)
+    notes_before: list[str] = dataclasses.field(default_factory=list)
+    provenance: Provenance | None = None
+    readback: bool = False
+    kept: bool = False
+    lists: dict = dataclasses.field(default_factory=dict)
+    system: str = ""
+    stable_extra: str = ""