mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/frontier/stages/after.py

@@ -0,0 +1,192 @@
+"""The stages that run once an action was sent: the screen after, the record, and whether the chain goes on."""
+
+import time
+
+from ... import contract_asks as proof_asks
+from ... import contract_screen as proof_screen
+from ...agent_hooks import GuardVerdict
+from ..observe import TWO_READ_OPS, UNPROVEN_SETTLES
+from ..prompt import row_texts
+from ..protocol import HOUSEKEEPING_OPS, SCROLL_OPS, TYPED_OPS
+from ..safety import COVERED, PENDING_REASON, SAME_FAILURES_TO_CHECK, GuardStop, consequential, input_shows
+from ..screen import names_act, screen_signature, text_status
+
+LOG_LABEL_CHARS, LOG_TEXT_CHARS = 200, 600
+
+
+def settled_rect(op, element, snapshot):
+    """Where the element this step acted on sits on the screen the step ends on, as fractions ``[x, y, w, h]``, for a
+    tap or typing step whose element is still there (the same locator, role and label). None when it is unknown."""
+    if element is None or not (op in TYPED_OPS or op in ("TAP", "LONG_PRESS")):
+        return None
+    for other in getattr(snapshot, "elements", None) or ():
+        if (other.locator and other.locator == element.locator and other.role == element.role
+                and other.label == element.label):
+            try:
+                x, y, w, h = (float(value) for value in other.rect)
+            except (TypeError, ValueError):
+                return None
+            return [round(x, 4), round(y, 4), round(w, 4), round(h, 4)] if w > 0 and h > 0 else None
+    return None
+
+
+def may_commit(op, element, snapshot):
+    """Whether a press may have committed something, by the loop's own test. A test that fails counts as a commit."""
+    try:
+        return consequential(op, element, snapshot)
+    except Exception:
+        return True
+
+
+def action_label(op, element, app):
+    if op in ("SWIPE_LEFT", "SWIPE_RIGHT"):
+        return element.label[:120] if element is not None and element.label else None
+    return element.label[:120] if element else app
+
+
+def settle(agent, attempt, turn):
+    index = turn.index
+    op, element, gate, before = attempt.op, attempt.element, attempt.gate, attempt.before
+    attempt.exec_ms = agent._spend("exec", attempt.t_exec)
+    t_settle = time.monotonic()
+    following = turn.chunk[index + 1].op if index + 1 < len(turn.chunk) else None
+    last = following in (None, "DONE", "BLOCKED")
+    # HOME right before LAUNCH_APP uses nothing on the home screen, so its settle is skipped.
+    passing = op == "HOME" and following == "LAUNCH_APP"
+    if passing:
+        turn.snapshot = before
+    elif op in TYPED_OPS and following == "SUBMIT":
+        # Return goes to the field just typed into: one read finds it.
+        turn.snapshot = agent._read_once()
+    else:
+        turn.snapshot = agent._settle(before, last and op not in TWO_READ_OPS, rest=last and op == "LAUNCH_APP")
+    turn.unproven = last and getattr(agent.driver, "settled_by", None) in UNPROVEN_SETTLES
+    attempt.settle_ms = agent._spend("settle", t_settle)
+    if agent._contract is not None:
+        agent._contract.observe(agent._masked(turn.snapshot), agent._app_name(turn.snapshot), turn.step)
+    attempt.changed = passing or turn.snapshot.content_fingerprint != before.content_fingerprint
+    if (not attempt.changed and gate is not None and gate.item is not None and proof_asks.asks(gate.item)
+            and agent.permissions is not None and gate.item.status != "closed"
+            and str(gate.item.count or "1").strip() in ("", "1")):
+        # Sent, no receipt yet: the same act again asks first, with the warning, in every mode.
+        agent.run_state.held_items[gate.item.id] = PENDING_REASON
+    gone = agent._row_gone(op, element, before, turn.snapshot)
+    if gone is not None:
+        turn.result = {"status": "blocked", "answer": None, "code": "unexpected_change",
+                            "reason": gone}
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "row_gone"})
+        return "end"
+    if op in SCROLL_OPS and following in SCROLL_OPS and attempt.changed:
+        # The next swipe was chosen without this screen: its rows would scroll past unseen (mem-009).
+        if turn.turn_rows is None:
+            turn.turn_rows = set(row_texts(agent._masked(turn.turn_screen)))
+        for row in row_texts(agent._masked(turn.snapshot)):
+            if row not in turn.turn_rows:
+                turn.turn_rows.add(row)
+                agent.run_state.passed.append(row)
+    done_key = agent._act_key(op, element, before) if attempt.changed else None
+    if agent.run_state.act_seen:
+        where, now = agent._app_name(turn.snapshot), None
+        for (act_app, _, _), looked in agent.run_state.act_seen.items():
+            if act_app == where:
+                now = now or screen_signature(turn.snapshot)
+                looked.add(now)
+    if done_key is not None:
+        agent.run_state.acts_done[done_key] = turn.step
+        agent.run_state.act_left[done_key] = screen_signature(turn.snapshot)
+        agent.run_state.act_seen[done_key] = {agent.run_state.act_left[done_key]}
+        agent.run_state.act_refused.pop(done_key, None)
+
+
+def record(agent, attempt, turn):
+    index = turn.index
+    op, text, app, element, gate, before, changed = (
+        attempt.op, attempt.text, attempt.app, attempt.element, attempt.gate, attempt.before, attempt.changed)
+    turn.after = op if changed or op in TYPED_OPS else None
+    turn.last_done = (attempt.key, before.content_fingerprint, turn.snapshot.content_fingerprint)
+    turn.recent.append(attempt.spot)
+    agent.last_screen = turn.snapshot
+    agent._emit({"event": "frontier_action", "step": turn.step, "index": index, "operation": op,
+               "target_label": action_label(op, element, app), "changed": changed,
+               "text": agent._mask(text[:200]) if text is not None and (op in TYPED_OPS or op == "SCROLL_TO") else None,
+               "field": ("search" if proof_screen.is_search_field(element) else "composer"
+                         if proof_screen.COMPOSER.search(element.label or "") else "field")
+               if element is not None and op in TYPED_OPS else None,
+               "app_name": agent._app_name(before) or None,
+               # Whether a press may have committed something: the step's commit.
+               **({"consequential": may_commit(op, element, before)} if op in ("TAP", "LONG_PRESS") else {}),
+               **({"element_rect": rect} if (rect := settled_rect(op, element, turn.snapshot)) else {}),
+               "refresh_ms": attempt.refresh_ms, "exec_ms": attempt.exec_ms, "settle_ms": attempt.settle_ms,
+               **({"settle_prediction": ps} if (ps := getattr(agent.driver, "predicted_settle", None)) else {})})
+    what = (element.label[:LOG_LABEL_CHARS] if element else app or "")
+    receipt = agent.run_state.last_receipt
+    if receipt is not None:
+        agent._emit({"event": "frontier_text", "step": turn.step, "index": index, "method": receipt.get("method"),
+                    "chars": receipt.get("chars"), "matches": receipt.get("matches"),
+                    "repaired": receipt.get("repaired"), "flattened": receipt.get("flattened")})
+    turn.history.append(f"{turn.step}.{index}: {op} {what!r}" + (f" text={text[:LOG_TEXT_CHARS]!r}" if text else "")
+                   + (" -> screen changed" if changed else " -> NO visible change")
+                   + (f"; field: {text_status(receipt)[:160]}" if receipt is not None else "")
+                   + ("; Return NOT pressed" if receipt is not None and op == "TYPE_SUBMIT"
+                      and receipt.get("submitted") is False else ""))
+    if agent.run_state.readback:
+        if changed and op in ("TAP", "LONG_PRESS", "TAP_XY") and names_act(op, element, text):
+            said = agent._read_back(turn.snapshot)
+            if said:
+                turn.history[-1] += "; " + said
+        agent._remember_lists(turn.snapshot)
+    if op == "SET_TEXT" and text and not changed and not input_shows(element, text):
+        # Only an unchanged SET_TEXT stops what follows: a TYPE's text is checked by its receipt.
+        turn.untaken = (op, text)
+    if agent.step_observer is not None:
+        agent._observe_step(turn.step, index, op, before, element, text, changed, gate)
+    else:
+        agent.run_state.asked = None
+    agent._check_app(turn.snapshot)
+    if gate is not None and gate.item is not None and gate.item.act == "install":
+        # The App Store's own confirmation is the person's: the guard hands it off or stops (C3).
+        verdict = agent._guard_check("sheet_suspected", turn.history)
+        if verdict is not None and verdict.state == "unlocked":
+            turn.snapshot = agent._observe()
+    if changed:
+        turn.covered = 0
+        turn.failures.clear()  # only failures in a row stop the run
+    if (not changed and op in ("TAP", "TAP_XY", "LONG_PRESS") and not turn.checked_front
+            and agent._front_app_differs(turn.snapshot)):
+        # The tree is the app's, but SpringBoard is in front: the lock screen or a system sheet.
+        turn.checked_front = True
+        verdict = agent._guard_check("lock_suspected", turn.history)
+        if verdict is not None and verdict.state == "unlocked":
+            turn.snapshot = agent._observe()
+        elif verdict is None:
+            # No guard to look: taps land on a screen the tree doesn't show, and the second time the run stops (B2).
+            turn.covered += 1
+            name = agent._app_name(turn.snapshot) or "the app"
+            if turn.covered >= SAME_FAILURES_TO_CHECK:
+                raise GuardStop(GuardVerdict("stop", "phone_locked", COVERED.format(app=name)))
+            turn.feedback = (f"The iPhone's lock screen, Home Screen or a system sheet is in front of {name}: "
+                        "what is listed may not be what the screen shows. If it is the lock screen or a "
+                        "passcode screen, finish with BLOCKED; never tap it.")
+        return "stop"
+
+
+def chain(agent, attempt, turn):
+    index = turn.index
+    op, element, changed = attempt.op, attempt.element, attempt.changed
+    if attempt.commit_now and index + 1 < len(turn.chunk) and turn.chunk[index + 1].op not in ("DONE", "BLOCKED"):
+        # A commit ends its chain: the next action is chosen on the screen it left.
+        agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index, "reason": "commit"})
+        return "stop"
+    if not changed and op in HOUSEKEEPING_OPS and [a.op for a in turn.chunk[index + 1:]] == ["DONE"]:
+        # Closing or scrolling had nothing to do; the DONE after it stands (mem-004).
+        return "next"
+    if (not changed and op == "TAP" and element is not None and element.editable
+            and index + 1 < len(turn.chunk) and turn.chunk[index + 1].op in TYPED_OPS):
+        # Focusing a field often shows nothing; the typing after it is the point (multi-010).
+        return "next"
+    if not changed and op not in TYPED_OPS:
+        if index + 1 < len(turn.chunk):
+            agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                       "reason": "no_change"})
+        return "stop"  # an action with no effect: look again before going on
+    return "next"

mobile_agent/frontier/stages/approve.py

@@ -0,0 +1,138 @@
+"""The stage that decides who asks for an action, and what follows the answer."""
+
+from ... import contract_asks as proof_asks
+from ...agent_hooks import RESUME_AFTER_SECONDS
+from ..interrupts import Where, check
+from ..protocol import ELEMENT_OPS
+from ..safety import (
+    CAP_REASON,
+    CHANGED_WHILE_ASKING,
+    INJECTION_REASON,
+    MAX_COMMITS_PER_RUN,
+    PENDING_REASON,
+    SAME_REFUSALS_TO_STOP,
+    UNKNOWN_REASON,
+    _nobody,
+    binding,
+    declined_result,
+    refused_sentence,
+    same_binding,
+)
+
+
+def approval(agent, attempt, turn):
+    index = turn.index
+    op, text, unknown_repeat, gate = attempt.op, attempt.text, attempt.unknown_repeat, attempt.gate
+    asker, force = agent.approve, False
+    agent.run_state.ask_reason = None
+    attempt.commit_now = agent._is_commit(gate, op, attempt.element, turn.snapshot, text)
+    held_reason = (agent.run_state.held_items.get(gate.item.id) if gate is not None and gate.item is not None
+                   and gate.item.status != "closed" else None)
+    if (held_reason is None and gate is not None and gate.item is not None and gate.item.status != "closed"
+            and gate.item.act in agent.run_state.held_acts and proof_asks.asks(gate.item)):
+        held_reason = UNKNOWN_REASON  # an earlier run of this task may have done it already
+    extra = None
+    if held_reason is not None and unknown_repeat is None:
+        if (agent.approve or agent.forced_approve or agent.floor_approve) is None and agent.permissions is None:
+            if held_reason == UNKNOWN_REASON:
+                turn.feedback = (f"Refused: an earlier {gate.item.act} for this request has an unknown outcome: "
+                            "it may have happened. Doing it again needs the user's OK, and nobody can "
+                            "give it here.")
+                turn.history.append(f"{turn.step}.{index}: {op} {attempt.element.label[:60]!r} refused "
+                                    "(an earlier attempt's outcome is unknown)")
+                agent._emit({"event": "frontier_refused", "step": turn.step, "label": "outcome_unknown"})
+                return "stop"
+        else:
+            extra = held_reason
+    elif (attempt.commit_now or agent._out_of_scope(op, attempt.element, turn.request)) and agent.run_state.injected:
+        # A screen this run tried to steer Mobster: every commit and every change no request named asks.
+        extra = INJECTION_REASON
+    elif attempt.commit_now and agent.run_state.commits >= MAX_COMMITS_PER_RUN:
+        extra = CAP_REASON.format(agent.run_state.commits)
+    if extra is not None and attempt.element is not None:
+        force, agent.run_state.ask_reason = True, extra
+        asker = asker or agent.forced_approve or agent.floor_approve or (
+            _nobody if agent.permissions is not None or extra in (UNKNOWN_REASON, PENDING_REASON) else None)
+    if turn.session is not None and attempt.element is not None and agent._long_asks(gate, op, attempt.element):
+        # A long run asks before every act that asks, Ask before acting off too (P1-2).
+        force = True
+    if force and attempt.element is not None:
+        if asker is None:
+            asker = agent.forced_approve
+        if asker is None:
+            turn.feedback = (f"Refused: {attempt.element.label[:60]!r} needs the user's OK, and this run can't ask for "
+                        "it. Finish with what you found, or BLOCKED if the request needs this.")
+            turn.history.append(
+                f"{turn.step}.{index}: {op} {attempt.element.label[:60]!r} refused (needs the user's OK)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "needs_approval"})
+            return "stop"
+    if asker is None and agent.permissions is not None:
+        # Full access: what asks in every mode (a one-time code) still reaches the person.
+        asker = agent.floor_approve
+    if asker is None and agent._asks_mode():
+        # A mode that asks with nobody to answer: what would ask is refused, never done.
+        asker = _nobody
+    cards_before = agent.run_state.cards
+    if unknown_repeat == "ask":
+        # A consequential action whose first outcome is unknown asks in every mode.
+        asker = agent.approve or agent.forced_approve or agent.floor_approve
+    if asker is not None:
+        verdict, waited = agent._approval(gate, op, attempt.element, turn.snapshot, text, turn.chain_typed, turn.step,
+                                          asker=asker, force=force, unknown=unknown_repeat == "ask")
+        turn.started += waited  # waiting for the user never spends the task's time
+        agent._waited(waited)
+        if verdict is None and check(agent, turn, Where.WAITED) is not None:
+            # Approved, but Stop came too: a stopped task never acts on an approval.
+            return "end"
+        if verdict is None and agent.run_state.cards == cards_before and op in ("TAP", "LONG_PRESS") \
+                and agent._asks_mode():
+            # The two-headed check: the pixels in the control's frame against its label.
+            verdict, more = agent._two_heads(asker, op, attempt.element, turn.snapshot, turn.chain_typed, turn.step)
+            turn.started += more
+            agent._waited(more)
+        if verdict is not None and verdict.startswith("refused:"):
+            turn.feedback = (f"Refused: {attempt.element.label[:60]!r} needs the user's OK, and nobody can approve it "
+                        "here. Finish with what you found, or BLOCKED if the request needs this.")
+            turn.history.append(
+                f"{turn.step}.{index}: {op} {attempt.element.label[:60]!r} refused (needs the user's OK)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "needs_approval"})
+            turn.refusals[turn.feedback] = turn.refusals.get(turn.feedback, 0) + 1
+            if turn.refusals[turn.feedback] >= SAME_REFUSALS_TO_STOP:
+                turn.result = {"status": "blocked", "answer": None, "code": "stuck",
+                          "reason": refused_sentence(turn.feedback, turn.refusals[turn.feedback], turn.actions)}
+                return "end"
+            return "stop"
+        if verdict is not None and verdict.startswith("redirected:"):
+            agent.run_state.asked = "redirected"
+            instruction = " ".join(verdict.split(":", 1)[1].split())[:500]
+            turn.request = f"{turn.request}\n\nThe user's later instruction: {instruction}"
+            turn.feedback = (f"The user declined {attempt.element.label[:60]!r} and asked instead: {instruction!r}. "
+                        "Do that now; change what you typed if it needs changing.")
+            turn.history.append(f"{turn.step}.{index}: {op} {attempt.element.label[:60]!r} declined; user redirected")
+            return "stop"
+        if verdict is not None:
+            turn.result = declined_result(verdict)
+            return "end"
+        if waited >= RESUME_AFTER_SECONDS:
+            # The phone may have locked while the person decided.
+            resumed = agent._guard_check("resume", turn.history)
+            if resumed is not None and resumed.state == "unlocked" and attempt.element is not None:
+                turn.snapshot = agent._observe()
+                attempt.element = agent._find(
+                    turn.snapshot, attempt.element.label, attempt.element.role, attempt.element.locator)
+                if attempt.element is None:
+                    turn.feedback = "The screen changed while Mobster waited for your OK: look again."
+                    return "stop"
+        if agent.run_state.cards != cards_before and attempt.element is not None and op in ELEMENT_OPS:
+            # The approval is bound to what the card showed: any change on a fresh read means no action now.
+            bound = binding(turn.snapshot, attempt.element)
+            fresh = agent._read_once()
+            found = agent._find(fresh, attempt.element.label, attempt.element.role, attempt.element.locator)
+            if found is None or not same_binding(bound, binding(fresh, found)):
+                turn.feedback = CHANGED_WHILE_ASKING
+                turn.history.append(f"{turn.step}.{index}: {op} {attempt.element.label[:60]!r} not done "
+                                    "(the screen changed while asking)")
+                agent._emit({"event": "frontier_refused", "step": turn.step, "label": "changed_while_asking"})
+                turn.snapshot, turn.unproven = fresh, False
+                return "stop"
+            turn.snapshot, attempt.element = fresh, found

mobile_agent/frontier/stages/controls.py

@@ -0,0 +1,115 @@
+"""The stages that find the control an action names and check its shape."""
+
+import dataclasses
+
+from ...agent_hooks import safe_allows
+from ...state import ADJUSTABLE_ROLES, TEXT_OPERATIONS, pixel_read
+from ..act import opens_field
+from ..protocol import ELEMENT_OPS, TYPED_OPS
+from ..safety import GuardStop, holds_text, revealed_actions, row_under_stroke
+from ..screen import editable_in
+
+
+def target(agent, attempt, turn):
+    index = turn.index
+    if attempt.op in ELEMENT_OPS:
+        attempt.element = agent._target(attempt.act, index, turn.aliases, turn.snapshot, turn.after)
+        if attempt.element is None:
+            agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                       "reason": "target_missing", "wanted": (attempt.act.target_label or "")[:80]})
+            if index == 0:
+                turn.feedback = f"{attempt.op} needs a target element id from the current screen."
+            wanted = attempt.act.target_label or attempt.act.target or attempt.act.opened_by
+            turn.history.append(f"{turn.step}.{index}: {attempt.op} target {wanted!r} "
+                           + ("(the field it opened) " if attempt.act.opened_by else "")
+                           + "not on screen; stopped the sequence")
+            return "stop"
+        if attempt.op in TYPED_OPS and opens_field(attempt.element, turn.snapshot):
+            # Typing into a field's prompt drawn as a button fails: it is tapped first, with every check a TAP gets.
+            agent._emit({"event": "frontier_repaired", "step": turn.step, "index": index, "operation": attempt.op,
+                        "as": "TAP", "label": agent._mask(attempt.element.label)[:80]})
+            turn.chunk.insert(index + 1, dataclasses.replace(attempt.act, target=None, target_label=None,
+                                                              opened_by=attempt.element.label))
+            attempt.act = dataclasses.replace(attempt.act, op="TAP", text=None)
+            attempt.op, attempt.text = "TAP", None
+
+
+def sideways_swipe(agent, attempt, turn):
+    index = turn.index
+    act, op = attempt.act, attempt.op
+    if op in ("SWIPE_LEFT", "SWIPE_RIGHT"):
+        # A sideways swipe strokes through the row it targets (B7); untargeted over a list row it is refused.
+        named = act.target or act.target_label
+        attempt.element = agent._target(act, index, turn.aliases, turn.snapshot, turn.after) if named else None
+        if named and attempt.element is None:
+            agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                        "reason": "target_missing", "wanted": str(named)[:80]})
+            turn.history.append(f"{turn.step}.{index}: {op} target {named!r} not on screen; stopped the sequence")
+            return "stop"
+        if (attempt.element is not None and (attempt.element.role == "Cell" or pixel_read(turn.snapshot))
+                and revealed_actions(turn.snapshot, attempt.element)):
+            # Its actions already show: another swipe could run the app's full swipe (Mail trashes).
+            turn.feedback = (f"Refused: {agent._mask(attempt.element.label)[:60]!r} already shows its actions. "
+                             "TAP the one the request needs, or leave it.")
+            turn.history.append(f"{turn.step}.{index}: {op} refused (its row's actions already show)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "swipe_again"})
+            return "stop"
+        crossed = row_under_stroke(turn.snapshot) if attempt.element is None else None
+        if crossed is not None:
+            turn.feedback = (f"Refused: {op} needs a target here: name the row to swipe (its id), or it "
+                        f"would swipe whichever row is in the middle of the screen "
+                        f"({agent._mask(crossed.label)[:60]!r}).")
+            turn.history.append(f"{turn.step}.{index}: {op} refused (no target row)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "untargeted_swipe"})
+            return "stop"
+
+
+def shape(agent, attempt, turn):
+    index = turn.index
+    app = attempt.app
+    clearing = attempt.op == "SET_TEXT" and attempt.text == "" and attempt.element is not None and (
+        agent._exact_text() or callable(getattr(agent.driver, "clear_text", None)))
+    if (attempt.op in TYPED_OPS or attempt.op == "SCROLL_TO") and not attempt.text and not clearing:
+        turn.feedback = f"{attempt.op} needs text." + (
+            ' SET_TEXT with text "" empties a field.' if attempt.op in TYPED_OPS else "")
+        return "stop"
+    if attempt.op in ("TAP", "LONG_PRESS") and attempt.element is not None and not attempt.element.enabled:
+        turn.feedback = (f"Refused: {attempt.element.label[:60]!r} is disabled: it does nothing yet. "
+                         "Do what enables it first (fill the required fields, choose an option).")
+        turn.history.append(f"{turn.step}.{index}: {attempt.op} {attempt.element.label[:60]!r} refused (disabled)")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "disabled"})
+        return "stop"
+    if attempt.op == "LAUNCH_APP" and app not in agent.apps:
+        turn.feedback = "LAUNCH_APP needs one of the listed apps."
+        return "stop"
+    if attempt.op == "LAUNCH_APP" and not safe_allows(agent.guard, app):
+        instruction = agent._app_instruction()
+        if not instruction:
+            raise GuardStop(agent._blocked_app(app))
+        turn.request = f"{turn.request}\n\nThe user's later instruction: {instruction}"
+        turn.feedback = (f"The user didn't allow {agent.apps.get(app, app)} and asked instead: {instruction!r}. "
+                    "Do that now.")
+        turn.history.append(f"{turn.step}.{index}: LAUNCH_APP {app} declined; user redirected")
+        return "stop"
+    if attempt.op in TYPED_OPS and attempt.element is not None and not attempt.element.editable \
+            and attempt.element.role not in ADJUSTABLE_ROLES:
+        # A row that holds a field is not the field.
+        attempt.element = editable_in(turn.snapshot, attempt.element) or attempt.element
+    if attempt.text and attempt.element is not None and agent._exact_text():
+        # write_text keeps line breaks where the field takes them and never types Return; other controls are spaces.
+        attempt.text = "".join(ch if ch.isprintable() or ch == "\n" else " " for ch in attempt.text)
+    elif attempt.text and attempt.element is not None and attempt.element.role != "TextView":
+        # Line breaks are text only in a text view; elsewhere Return would submit.
+        attempt.text = " ".join("".join(ch if ch.isprintable() else " " for ch in attempt.text).split())
+    if (attempt.op in TEXT_OPERATIONS and attempt.element is not None and attempt.element.editable
+            and holds_text(attempt.element, attempt.text)
+            and (attempt.op == "TYPE" or agent._appends(attempt.op, attempt.element))):
+        # Text a field already shows isn't typed again; a TYPE_SUBMIT that would append only presses Return.
+        turn.feedback = f"{attempt.element.label[:60]!r} already held that text, so it wasn't typed again."
+        turn.history.append(f"{turn.step}.{index}: {attempt.op} {attempt.element.label[:60]!r} not typed: "
+                            "the field already held the text")
+        agent._emit({"event": "frontier_skipped", "step": turn.step, "index": index, "operation": attempt.op,
+                    "reason": "already_typed"})
+        if attempt.op == "TYPE":
+            return "next"
+        attempt.op, attempt.text = "SUBMIT", None

mobile_agent/frontier/stages/ends.py

@@ -0,0 +1,102 @@
+"""The stages that end or hold an action: an input that changed nothing, DONE, BLOCKED and WAIT."""
+
+import time
+
+from ..prompt import DONE_CHECK
+from ..protocol import TYPED_OPS, WRAP_UP_SHARE
+
+IDLE_WAIT_LIMIT = 2
+
+
+def input_not_taken(agent, attempt, turn):
+    index = turn.index
+    op = attempt.op
+    if turn.untaken is not None and op not in TYPED_OPS:
+        # An input that changed nothing: what follows was planned on a value that isn't there.
+        turn.feedback = (f"Your {turn.untaken[0]} {agent._mask(turn.untaken[1])[:60]!r} changed nothing on screen, so "
+                    f"{op} did not run. Check the values on screen and set them before going on.")
+        turn.history.append(f"{turn.step}.{index}: {op} not run: the {turn.untaken[0]} before it changed nothing")
+        agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                    "reason": "input_not_taken"})
+        return "stop"
+
+
+def _said(attempt, turn):
+    return turn.out.get("answer") or attempt.text
+
+
+def done_gate(agent, attempt, turn):
+    if agent._contract is None:
+        return None
+    index = turn.index
+    wrap = (agent.max_cost_usd is not None and turn.spent is not None
+            and turn.spent + turn.reserve >= WRAP_UP_SHARE * agent.max_cost_usd)
+    accepted, reason, answer = agent._contract.done_gate(
+        _said(attempt, turn), 0 if wrap else agent.max_steps - turn.calls)
+    if not accepted:
+        turn.feedback = reason
+        turn.history.append(f"{turn.step}.{index}: DONE refused: items not shown on screen yet")
+        agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                     "reason": "proof_gate", "open": [i.id for i in agent._contract.items
+                                                      if i.reason and not i.unproven]})
+        return "stop"
+    agent._reveal_found(turn)
+    turn.result = {"status": "completed", "answer": answer}
+    if turn.data_json:
+        turn.result["data_json"] = turn.out.get("data_json")
+    return "end"
+
+
+# No contract: a DONE right after a change, or before the request was walked item by item, is deferred once.
+def done_deferred(agent, attempt, turn):
+    if turn.calls >= agent.max_steps - 1 or (
+            agent.max_cost_usd is not None and turn.spent is not None
+            and turn.spent >= WRAP_UP_SHARE * agent.max_cost_usd):
+        return None
+    index = turn.index
+    reasons = ["You finished right after changing things: check on this screen that every "
+               "change took effect as intended."] if turn.acted else []
+    if not turn.request_checked:
+        reasons.append(DONE_CHECK)
+    turn.request_checked = True
+    if reasons:
+        turn.feedback = " ".join(reasons)
+        turn.history.append(f"{turn.step}.{index}: DONE deferred: check the result and the request first")
+        agent._emit({"event": "frontier_chunk_stop", "step": turn.step, "index": index,
+                     "reason": "verify_before_done"})
+        return "stop"
+
+
+def finish(agent, attempt, turn):
+    said = _said(attempt, turn)
+    if attempt.op == "BLOCKED":
+        sentence = said or (turn.out.get("thought") or "").strip() or None
+        turn.result = {"status": "blocked", "answer": sentence, "reason": sentence}
+    else:
+        turn.result = {"status": "completed", "answer": said}
+        if turn.data_json:
+            turn.result["data_json"] = turn.out.get("data_json")
+    return "end"
+
+
+def wait(agent, attempt, turn):
+    index = turn.index
+    if turn.idle_waits >= IDLE_WAIT_LIMIT:
+        turn.feedback = ("Refused: you have waited twice and nothing on screen changed; nothing is "
+                         "loading. Act instead: the thing you wait for may not exist (check that "
+                         "earlier steps were really saved), or try another route.")
+        turn.history.append(f"{turn.step}.{index}: WAIT refused (screen idle)")
+        agent._emit({"event": "frontier_refused", "step": turn.step, "label": "idle_wait"})
+        return "stop"
+    turn.history.append(f"{turn.step}.{index}: WAIT")
+    t_wait = time.monotonic()
+    time.sleep(1.0)
+    fresh = agent._observe()
+    agent._spend("wait", t_wait)
+    turn.idle_waits = turn.idle_waits + 1 if fresh.content_fingerprint == turn.snapshot.content_fingerprint else 0
+    turn.snapshot, turn.unproven = fresh, False
+    return "stop"
+
+
+def active(agent, attempt, turn):
+    turn.idle_waits = 0

mobile_agent/frontier/stages/gate.py

@@ -0,0 +1,92 @@
+"""The stages that ask the task's contract and the signed-in profile before an action."""
+
+from ... import contract_asks as proof_asks
+from ... import contract_screen as proof_screen
+from ... import harness_api
+from ...agent_hooks import GuardVerdict
+from ..actions import Action
+from ..safety import SAME_REFUSALS_TO_STOP, GuardStop, refused_sentence
+from ..screen import screen_image
+
+
+def contract_gate(agent, attempt, turn):
+    index = turn.index
+    op, element = attempt.op, attempt.element
+    attempt.gate = None
+    if agent._contract is not None and element is not None:
+        attempt.gate = agent._contract.check(op, element, turn.snapshot, agent._app_name(turn.snapshot),
+                                             text=attempt.text, chain_typed=tuple(turn.chain_typed))
+        if not attempt.gate.allowed:
+            turn.feedback = attempt.gate.reason
+            turn.history.append(f"{turn.step}.{index}: {op} {element.label[:60]!r} refused (not declared)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": element.label[:120],
+                        "reason": attempt.gate.reason[:200]})
+            turn.refusals[attempt.gate.reason] = turn.refusals.get(attempt.gate.reason, 0) + 1
+            if turn.refusals[attempt.gate.reason] >= SAME_REFUSALS_TO_STOP:
+                refused = turn.refusals[attempt.gate.reason]
+                agent._emit({"event": "frontier_stuck", "step": turn.step, "refused": refused,
+                             "reason": attempt.gate.reason[:200]})
+                turn.result = {"status": "blocked", "answer": None, "code": "stuck",
+                               "reason": refused_sentence(attempt.gate.reason, refused, turn.actions)}
+                return "end"
+            return "stop"
+    if (attempt.unknown_repeat == "retry" and attempt.gate is not None and attempt.gate.item is not None
+            and proof_asks.asks(attempt.gate.item)):
+        # The contract calls it a commit, so the person is asked again.
+        if (agent.approve or agent.forced_approve or agent.floor_approve) is None:
+            turn.feedback = (f"Refused: your earlier {op} {element.label[:60]!r} has an unknown outcome: it may "
+                        "have happened. Sending it again needs the user's OK, and nobody can give it here.")
+            turn.history.append(f"{turn.step}.{index}: {op} {element.label[:60]!r} refused (its earlier outcome was "
+                           "unknown)")
+            agent._emit({"event": "frontier_refused", "step": turn.step, "label": "outcome_unknown"})
+            return "stop"
+        attempt.unknown_repeat = "ask"
+
+
+def ask_at_send(agent, attempt, turn):
+    index = turn.index
+    text, element = attempt.text, attempt.element
+    can_ask = agent.approve is not None or (turn.session is not None and agent.forced_approve is not None)
+    if (attempt.op == "TYPE_SUBMIT" and can_ask and attempt.gate is not None
+            and attempt.gate.item is not None and proof_asks.asks(attempt.gate.item)
+            and attempt.gate.verbs == frozenset({"send"})):
+        # One message, one approval: type now, unasked; the Return asks with the typed text in its composer.
+        turn.chunk.insert(index + 1, Action("SUBMIT", target_label=element.label, locator=element.locator,
+                                             role=element.role))
+        attempt.op, attempt.gate = "TYPE", proof_screen.Gate(True)
+        attempt.key = (attempt.op, element.label, text)
+        attempt.spot = (attempt.op, element.locator or element.label, text, turn.snapshot.content_fingerprint)
+        agent._emit({"event": "frontier_split", "step": turn.step, "index": index, "reason": "ask_at_send"})
+
+
+def account(agent, attempt, turn):
+    index = turn.index
+    op, element, gate = attempt.op, attempt.element, attempt.gate
+    if agent.commit_check is not None and element is not None:
+        commit = (gate.item.act if gate is not None and gate.item is not None and proof_asks.asks(gate.item)
+                  else None) if agent._contract is not None else harness_api.commit_act(
+            op, element.label, None, getattr(turn.snapshot, "bundle_id", None))
+        agent.run_state.acting_as = agent.run_state.media_checked = None
+        if commit:
+            checked = agent.commit_check(commit, turn.snapshot) or {}
+            if checked.get("stop"):
+                agent._emit({"event": "frontier_refused", "step": turn.step, "label": element.label[:120],
+                            "reason": "account"})
+                raise GuardStop(GuardVerdict("stop", checked.get("code") or "account_mismatch",
+                                             checked["stop"]))
+            if checked.get("refuse"):
+                turn.feedback = checked["refuse"]
+                turn.history.append(f"{turn.step}.{index}: {op} {element.label[:60]!r} refused (profile not "
+                               "confirmed)")
+                agent._emit({"event": "frontier_refused", "step": turn.step, "label": element.label[:120],
+                            "reason": "account"})
+                return "stop"
+            agent.run_state.acting_as = checked.get("acting_as")
+            if callable(checked.get("preview")):
+                # A check of the screen this commit is made on (the queue's media check): no picture, or any failure,
+                # is no match.
+                image = screen_image(agent.driver, getattr(turn.snapshot, "read_started", None))
+                try:
+                    agent.run_state.media_checked = bool(image is not None and checked["preview"](image))
+                except Exception:  # noqa: BLE001
+                    agent.run_state.media_checked = False