tapwright 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tapwright/daemon.py ADDED
@@ -0,0 +1,2401 @@
1
+ """The phone side of `tapwright serve`: one Session (tunnel and WebDriverAgent), the control
2
+ rules, the stream, the agent tools, a person's input from the live view, and a supervisor
3
+ that keeps it all connected. server.py only turns HTTP into calls here."""
4
+ import asyncio
5
+ import base64
6
+ import io
7
+ import json
8
+ import dataclasses
9
+ import math
10
+ import os
11
+ import re
12
+ import time
13
+ import traceback
14
+ from contextlib import nullcontext
15
+ from typing import NamedTuple
16
+
17
+ from PIL import Image, UnidentifiedImageError
18
+
19
+ from . import changes, ui
20
+ from . import session as session_module
21
+ from . import shot as shotlib
22
+ from .control import TAP_MS, Controller, Refused, classify
23
+ from .device import (AppNotAnswering, ConnectionLost, DeviceError, NotSetUp, PasscodeNeeded, PhoneLocked, Unanswered,
24
+ PhoneNotFound)
25
+ from .session import Snapshot
26
+ from .tapline import AgentTouches
27
+ from .stream import (NO_IDLE_WAIT, PRESETS, Frame, FrameHub, PictureFeeder, PictureGone, StreamPump, alike, jpeg_of,
28
+ turn_jpeg)
29
+
30
+ PROTOCOL = 1
31
+ ACCEPTS = ["tap", "longPress", "drag", "scroll", "text", "enter", "backspace", "home", "volume", "quality"]
32
+ BUTTONS = ("home", "volumeUp", "volumeDown")
33
+ STILL = 0.1 # seconds without a changed frame before a screen counts as still (docs/AGENT-TOOLS.md §6)
34
+ RESOLVE_SETTLE = 1.0 # the longest a silent re-read waits for the screen to be still
35
+ CAP = 2.0 # the longest an action's answer waits for the screen
36
+ LONG_CAP = 4.0 # the same after launching an app or opening a URL
37
+ LIVE_AGE = 2.0 # seconds a list stays current for id, label and text targets (docs/AGENT-TOOLS.md §2)
38
+ FRESH_FRAME = 2.0 # seconds a stream frame counts as "the screen now"
39
+ MIN_QUIET = 0.3 # seconds a screen must stay as it was before "No visible change." is believed
40
+ NOTE_CAPPED = ("the screen was still changing after {cap:g} s (an animation or video); "
41
+ "positions may be mid-change.")
42
+
43
+
44
+ class Scope(NamedTuple):
45
+ """Personal mode: the apps an agent may read and touch, as bundle ids or exact names. Empty: none, so
46
+ nothing runs (it fails closed). A dedicated device has no Scope at all."""
47
+ apps: tuple[str, ...] = ()
48
+
49
+
50
+ NO_APPS = ("no_apps: this agent was started without an app list, so it may not use the phone. Start it with "
51
+ "`tapwright mcp --apps <bundle id>` for the apps under test, or with `--dedicated` on a device "
52
+ "kept for testing.")
53
+ SCOPE_FREE = ("open_app", "close_app", "open_url", "press_button") # how an agent gets back to its app
54
+ APP_LINE = re.compile(r"^App: (.+) \(([^()\s]+)\)$", re.M) # the header line of a full list: name, bundle id
55
+
56
+
57
+ def _outside(bundle: str | None, name: str | None, given: str) -> str:
58
+ front = "the phone can't say which app" if not bundle else f"{name} ({bundle})" if name else bundle
59
+ return (f"outside_apps: {front} is in front and isn't one of the apps you were given ({given}), so nothing of it is shown or "
60
+ "touched. Go back with open_app.")
61
+
62
+
63
+ class Settled(NamedTuple):
64
+ seconds: float
65
+ changed: bool # False only when the stream showed the screen unchanged for certain
66
+ capped: bool # the cap ended the wait with frames still changing
67
+ WAIT_MIN, WAIT_MAX = 0.1, 30.0 # seconds, for `wait`
68
+ WAIT_FOR_MIN, WAIT_FOR_DEFAULT = 0.5, 10.0 # `wait_for`'s timeout in seconds (its upper end is WAIT_MAX)
69
+ WAIT_FOR_KINDS = ("text", "id", "label")
70
+ WAIT_FOR_POLL = 0.3 # seconds between tree reads while waiting
71
+ SCROLL_MIN, SCROLL_MAX = 0.1, 1.0 # `scroll`'s amount: a share of the scroll area
72
+ MAX_SCROLL_SWIPES = 15 # `scroll to start|end` gives up after this many swipes
73
+ SCROLL_CAP = 12.0 # ... or after this many seconds (a spinner or a bounce can keep the screen moving)
74
+ SCROLL_TO = {"end": ("down", "right"), "start": ("up", "left")} # the directions each `to` can mean; the first is the default
75
+ SCROLL_TOP_INSET, SCROLL_BOTTOM_INSET = 60.0, 40.0 # points a swipe keeps off the screen's edges when no bar says more
76
+ WAIT_FOR_READ_MAX = 5.0 # the longest one tree read in `wait_for` may take
77
+ WAIT_FOR_MIN_LEFT = 0.1 # with less time than this left, `wait_for` gives up instead of reading again
78
+ WAIT_FOR_FIRST_READ = 3.0 # `wait_for`'s first read of the screen gets at least this long (twice the slowest look allowed)
79
+ # The stream's pace. Every frame WebDriverAgent captures costs the phone, mostly in iOS's testmanagerd: about 88% of
80
+ # a core at balanced's ~27 fps and 38% at light's ~10 fps (DVT sysmontap, the lab animating, 2026-11-10). So the
81
+ # stream runs at its preset's rate while something happens, and at STILL_FPS once nothing has: no agent call for
82
+ # the first quiet span (the stream's own linger, so an agent between steps never waits for the rate to rise), no
83
+ # person's input for the second, no change on the screen for the third. STILL_FPS keeps the longest gap
84
+ # between frames at 0.2 s (V3: at most 0.25 s).
85
+ STILL_FPS = 5
86
+ PACE_QUIET = (30.0, 2.0, 1.0) # seconds without an agent call, a person's input, a change on the screen
87
+ PACE_TICK = 0.25 # how often the pace is checked
88
+ LOOP_TICK, LOOP_HELD, LOOP_SAY_EVERY = 0.05, 0.3, 10.0 # seconds: see Daemon._watch_loop
89
+ BACK_STILL_PAUSE, BACK_STILL_TRIES = 0.3, 6 # see Daemon._still_back
90
+ TOGGLE_LAND_WAIT, TOGGLE_POLL = 0.5, 0.05 # seconds: see Daemon._toggle_landed
91
+ # A switch element this many times wider than tall is a whole row (label and control): toggle touches the control,
92
+ # SWITCH_FROM_RIGHT points in from its right end (half an iOS switch, 63 pt wide), not the label in the middle
93
+ WIDE_SWITCH, SWITCH_FROM_RIGHT = 3.0, 32.0
94
+ NOTE_PERSON = "a person used the phone since your last look."
95
+ # a notification banner took a touch near the top (Daemon._taken_away)
96
+ NOTE_TAKEN = ("{other} came to the front after the touch: the app opened it, or a notification banner over the top of "
97
+ "the screen took the touch. Went back to {app}. If the app was meant to open it, that worked; if not, "
98
+ "look, then try again.")
99
+ NOTE_TAKEN_SYSTEM = ("{other} came to the front after the touch: a notification banner over the top of the screen, or "
100
+ "a drag from its top edge, can take a touch there. Went back to {app}; the touch may not have "
101
+ "reached it. Look, then try again.")
102
+ SYSTEM_IN_FRONT = "the home screen, Notification Center or Control Center"
103
+ RECHECK_WITHIN = 5.0 # seconds: a banner's theft shows within about one; later, the agent or a person moved on
104
+ NOTE_KEYBOARD = "closed the keyboard first."
105
+ NOTE_RETYPED = "the field didn't hold what was typed (keys went missing), so it was cleared and typed once more."
106
+ SCROLL_HINT = 'scroll to it first (scroll with this target and to: "target").' # AGENT-TOOLS v1.6: the one-call way
107
+ NOTE_SPAN = "changes are since your last list, {n} actions ago." # after answers with look: false (v1.6)
108
+ # v1.6: `force` let through what the checks would have refused. A real touch doesn't make the hidden element act
109
+ NOTE_FORCED = "forced ({reason}): the touch may have landed on what is in front."
110
+ KEYBOARD_TOOLS = ("tap", "long_press", "toggle", "type_text") # the touches that close the keyboard before them
111
+ NOTE_COVERED = "something outside the app (a system sheet or browser) may cover it."
112
+
113
+
114
+ def _size(jpeg: bytes | None) -> list[int] | None:
115
+ """[width, height] in pixels of a JPEG, or None when there is no readable picture."""
116
+ if not jpeg:
117
+ return None
118
+ try:
119
+ with Image.open(io.BytesIO(jpeg)) as img:
120
+ return [img.width, img.height]
121
+ except (UnidentifiedImageError, OSError):
122
+ return None
123
+
124
+
125
+ MARK_NAME_MAX = 32 # characters of an element's name in the live view (VIEWER-PROTOCOL 1.4)
126
+ CAPTION_MAX = 60 # characters of a step's caption (VIEWER-PROTOCOL 1.4)
127
+ BUTTON_CAPTIONS = {"home": "Home", "volumeUp": "Volume up", "volumeDown": "Volume down"}
128
+
129
+
130
+ def element_name(result: str) -> str:
131
+ """The element's name in an action's first answer line ("Tapped e25 button "Equals" @320,780 95x90." →
132
+ `Equals`), cut to MARK_NAME_MAX; "" when it names none. The live view's `label` (VIEWER-PROTOCOL 1.4): the
133
+ whole answer line covered the keys around a touch and ran off the picture (recording the demo, 2026-11-11)."""
134
+ name = re.search(r'"([^"]+)"', result)
135
+ if name is None:
136
+ return ""
137
+ text = name.group(1)
138
+ return text if len(text) <= MARK_NAME_MAX else text[:MARK_NAME_MAX - 1] + "…"
139
+
140
+
141
+ def step_caption(tool: str, result: str, args: dict) -> str | None:
142
+ """The agent's step in plain words for people watching the live view (VIEWER-PROTOCOL 1.4, "Captions"):
143
+ `Tap "Sign up"`, `Type into "Email"`, `Scroll down`, `Open Wayfare`. Never the typed text, a link's address,
144
+ refs or coordinates. None for a step a viewer needn't see."""
145
+ name = element_name(result)
146
+ quoted = f'"{name}"' if name else ""
147
+ if tool in ("tap", "double_tap"):
148
+ text = f"Tap {quoted}"
149
+ elif tool == "long_press":
150
+ text = f"Press and hold {quoted}"
151
+ elif tool == "type_text":
152
+ text = f"Type into {quoted}" if name else "Type"
153
+ elif tool == "scroll":
154
+ to = args.get("to")
155
+ if to == "target" and name:
156
+ text = f"Scroll to {quoted}"
157
+ else:
158
+ text = f"Scroll {args.get('direction') or {'end': 'down', 'start': 'up'}.get(to or '', '')}"
159
+ elif tool == "swipe":
160
+ text = "Swipe"
161
+ elif tool == "toggle":
162
+ text = f"Switch {quoted}"
163
+ elif tool == "back":
164
+ text = "Go back"
165
+ elif tool == "alert":
166
+ text = "Answer the alert"
167
+ elif tool == "open_url":
168
+ text = "Open link" # never the address: a link can carry codes or tokens
169
+ elif tool in ("open_app", "close_app"):
170
+ app = re.match(r"(?:Opened|Closed) (.+?)(?: \(|\.|,|$)", result)
171
+ text = f"{'Open' if tool == 'open_app' else 'Close'} {app.group(1) if app else 'the app'}"
172
+ elif tool == "rotate":
173
+ text = "Turn the phone"
174
+ elif tool == "press_button":
175
+ text = BUTTON_CAPTIONS.get(str(args.get("name") or ""), "Press a button")
176
+ elif tool == "wait_for":
177
+ return None
178
+ else:
179
+ text = tool.replace("_", " ")
180
+ text = " ".join(text.split())
181
+ return text if len(text) <= CAPTION_MAX else text[:CAPTION_MAX - 1] + "…"
182
+
183
+
184
+ def _answer(lines: list[str], notes: list[str] | None = None, image: bytes | None = None) -> dict:
185
+ """One tool answer (docs/AGENT-TOOLS.md §3): the lines as one text, a `Note: ` line per note,
186
+ and the picture, if any, with its size."""
187
+ text = "\n".join([*lines, *(f"Note: {n}" for n in notes or [])])
188
+ return {"text": text, "image": base64.b64encode(image).decode() if image else None, "image_size": _size(image)}
189
+
190
+
191
+ def settled_line(seconds: float) -> str:
192
+ return f"Settled in {seconds:.2f} s."
193
+
194
+
195
+ def _alert_key(alert: ui.Alert | None) -> tuple[str, tuple[str, ...]] | None:
196
+ return (alert.title, alert.buttons) if alert else None
197
+
198
+
199
+ def _furniture(view, alert: ui.Alert | None, keyboard: bool) -> list[str]:
200
+ """What an alert or the keyboard did since the agent's last answer, as the first lines of a change block."""
201
+ lines = []
202
+ if _alert_key(alert) != view.alert:
203
+ lines.append(ui.alert_line(alert) if alert else "Alert: gone")
204
+ if keyboard != view.keyboard:
205
+ lines.append(f"Keyboard: {'shown' if keyboard else 'hidden'}")
206
+ return lines
207
+
208
+
209
+ def _number(args: dict, key: str) -> float | None:
210
+ """args[key] as a finite float; None when it isn't given. Anything else is bad_params."""
211
+ value = args.get(key)
212
+ if value is None:
213
+ return None
214
+ try:
215
+ number = float(value)
216
+ except (TypeError, ValueError):
217
+ number = math.nan
218
+ if not math.isfinite(number):
219
+ raise Refused("bad_params", f"{key} must be a number, not {value!r}")
220
+ return number
221
+
222
+
223
+ def _checked(name: str, args: dict) -> dict:
224
+ """The agent's arguments with numbers coerced, or Refused(bad_params) naming the one that's wrong.
225
+ This runs before anything reaches the phone, so a ValueError later is never the agent's fault."""
226
+ checked = dict(args)
227
+ for key in ("x0", "y0", "x1", "y1", "seconds", "amount"):
228
+ if (number := _number(args, key)) is not None:
229
+ checked[key] = number
230
+ if args.get("target") is not None:
231
+ checked["target"] = ui.parse_target(args["target"]) # bad_params, with an example of a target
232
+ if name == "scroll":
233
+ if checked.get("direction") is None: # null (the MCP server sends every argument) is left out
234
+ checked.pop("direction", None)
235
+ if checked.get("direction", "down") not in ("up", "down", "left", "right"):
236
+ raise Refused("bad_params", f"direction must be up, down, left or right, not {args['direction']!r}")
237
+ if args.get("to") not in (None, "start", "end", "target"):
238
+ raise Refused("bad_params", f"to must be start, end or target, not {args['to']!r}")
239
+ if args.get("to") == "target":
240
+ aim = checked.get("target")
241
+ if aim is None or aim.point is not None:
242
+ raise Refused("bad_params", 'to: "target" scrolls to an element: pass it as target (ref, id, label '
243
+ "or text)")
244
+ if "direction" in checked:
245
+ raise Refused("bad_params", 'to: "target" finds the direction itself; leave direction out')
246
+ elif args.get("to") and "direction" in checked and checked["direction"] not in SCROLL_TO[args["to"]]:
247
+ raise Refused("bad_params", f"to {args['to']} scrolls {' or '.join(SCROLL_TO[args['to']])}, not "
248
+ f"{args['direction']}; leave direction out, or make them agree")
249
+ if "amount" in checked and not SCROLL_MIN <= checked["amount"] <= SCROLL_MAX:
250
+ raise Refused("bad_params", f"amount must be from {SCROLL_MIN:g} to {SCROLL_MAX:g}, not {args['amount']!r}")
251
+ if name == "type_text":
252
+ if not isinstance(args.get("text"), str):
253
+ raise Refused("bad_params", f"text must be a string, not {args.get('text')!r}")
254
+ for key in ("replace", "submit", "force"):
255
+ if key in args and not isinstance(args[key], bool):
256
+ raise Refused("bad_params", f"{key} must be true or false, not {args[key]!r}")
257
+ if name == "rotate" and args.get("to") not in ("portrait", "landscape_left", "landscape_right"):
258
+ raise Refused("bad_params", f"to must be portrait, landscape_left or landscape_right, not {args.get('to')!r}")
259
+ if name == "wait_for":
260
+ given = [k for k in WAIT_FOR_KINDS if args.get(k) is not None]
261
+ if len(given) != 1:
262
+ raise Refused("bad_params", "wait_for needs exactly one of text, id or label")
263
+ if not isinstance(args[given[0]], str) or not args[given[0]]:
264
+ raise Refused("bad_params", f"{given[0]} must be a non-empty string, not {args[given[0]]!r}")
265
+ if "gone" in args and not isinstance(args["gone"], bool):
266
+ raise Refused("bad_params", f"gone must be true or false, not {args['gone']!r}")
267
+ timeout = args.get("timeout", WAIT_FOR_DEFAULT)
268
+ if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not WAIT_FOR_MIN <= timeout <= WAIT_MAX:
269
+ raise Refused("bad_params", f"timeout must be a number from {WAIT_FOR_MIN:g} to {WAIT_MAX:g}, not {timeout!r}")
270
+ for key in ("look", "screenshot", "labels"):
271
+ if key in args and not isinstance(args[key], bool):
272
+ raise Refused("bad_params", f"{key} must be true or false, not {args[key]!r}")
273
+ if name == "wait":
274
+ seconds = args.get("seconds") # required (REQUIRED); a bool is a number to Python, a string isn't one to us
275
+ if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or not WAIT_MIN <= seconds <= WAIT_MAX:
276
+ raise Refused("bad_params", f"seconds must be a number from {WAIT_MIN:g} to {WAIT_MAX:g}, not {seconds!r}")
277
+ if name in ("open_app", "close_app") and (not isinstance(args["app"], str) or not args["app"].strip()):
278
+ raise Refused("bad_params", f"app must be a bundle id or an app's name, not {args['app']!r}")
279
+ if name == "toggle" and args.get("value") is not None and not isinstance(args["value"], bool):
280
+ raise Refused("bad_params", f"value must be true or false, not {args['value']!r}")
281
+ return checked
282
+
283
+
284
+ class Noted(str):
285
+ """A handler's result line with a note for the answer's end."""
286
+ note: str = ""
287
+
288
+
289
+ class Unchanged(str):
290
+ """A handler's result that did nothing on the phone (`toggle` to the state it's already in): it is
291
+ the whole answer, with no change block."""
292
+
293
+
294
+ class Toggled(Noted):
295
+ """`toggle`'s result, `Tapped <line>.` until the tree is read: then the first line says what the switch did."""
296
+ ref: str = ""
297
+ before: ui.Element = ui.Element("", "switch") # the switch as it was tapped
298
+ want: bool | None = None # the `value` asked for
299
+ flipped: bool | None = None # where a tap should have taken it
300
+
301
+ def verdict(self, now: ui.Element, alert: bool) -> str:
302
+ """The first line from the switch as it reads after the tap. Raises when a wanted value wasn't reached
303
+ (unless an alert is up: it may be asking to confirm)."""
304
+ line = now.line(actions=False)
305
+ if now.checked == self.flipped:
306
+ return f"Toggled {line}."
307
+ if self.want is not None and not alert:
308
+ raise Refused("not_actionable", f"no_action: {line} didn't change")
309
+ return f"Tapped {line}, but it is still checked={'true' if now.checked else 'false'}."
310
+
311
+
312
+ class Typed(Noted):
313
+ """`type_text`'s result, `Typed into <line>.` until the tree is read: then the first line also gives the value."""
314
+ field: ui.Element = ui.Element("", "input") # the input as it was when typed into
315
+
316
+ def same_field(self, now: ui.Element) -> bool:
317
+ """Is `now` the field typed into (the ref may have gone to another input since)? The same ref and
318
+ role, and the same identifier, or without one the same label."""
319
+ old = self.field
320
+ return (now.ref, now.role, now.identifier) == (old.ref, old.role, old.identifier) \
321
+ and (bool(old.identifier) or now.label == old.label)
322
+
323
+ def verdict(self, now: ui.Element) -> str:
324
+ line = now.line(actions=False)
325
+ if now.secure:
326
+ return f"Typed into {line}. The value is hidden (secure field)."
327
+ return f"Typed into {line}. Value is now {json.dumps(now.value, ensure_ascii=False)}."
328
+
329
+
330
+ class Scrolled(Noted):
331
+ """`scroll`'s result, with a note for the answer when it stopped early."""
332
+
333
+
334
+ # What `open_app "Settings"` means: the built-in apps aren't in the installed list.
335
+ SYSTEM_APPS = {"Settings": "com.apple.Preferences", "Safari": "com.apple.mobilesafari",
336
+ "Calculator": "com.apple.calculator", "Camera": "com.apple.camera",
337
+ "Photos": "com.apple.mobileslideshow", "Messages": "com.apple.MobileSMS",
338
+ "Mail": "com.apple.mobilemail", "Maps": "com.apple.Maps", "App Store": "com.apple.AppStore",
339
+ "Clock": "com.apple.mobiletimer", "Notes": "com.apple.mobilenotes"}
340
+
341
+
342
+ def app_line(a: dict) -> str:
343
+ return f"{a['bundle_id']} {a['name']} {a['version']} ({a['build']})"
344
+
345
+
346
+ def _target(a: dict) -> ui.Target:
347
+ return a["target"] # parsed by _checked; tap and long_press require it
348
+
349
+
350
+ # Agent actions: tool -> (what it does on the session, seconds to settle before the look; None: no look)
351
+ ACTIONS = {
352
+ "open_app": (None, LONG_CAP), # None: the handler is a Daemon method (Daemon._handlers), which needs the agent's list
353
+ "close_app": (None, None),
354
+ "toggle": (None, CAP),
355
+ "back": (None, CAP),
356
+ "tap": (None, CAP),
357
+ "long_press": (None, CAP),
358
+ "type_text": (None, CAP),
359
+ "scroll": (None, CAP),
360
+ "swipe": (lambda s, a: s.swipe(float(a["x0"]), float(a["y0"]), float(a["x1"]), float(a["y1"]),
361
+ int(float(a.get("seconds", 0.35)) * 1000)), CAP),
362
+ "press_button": (lambda s, a: s.press_button(a["name"]), CAP),
363
+ "alert": (lambda s, a: s.alert(a["action"]), CAP),
364
+ "open_url": (lambda s, a: s.open_url(a["url"]), LONG_CAP),
365
+ "rotate": (lambda s, a: s.rotate(a["to"]), CAP),
366
+ }
367
+ SPRINGBOARD = "com.apple.springboard"
368
+ PROMPT_FRONT = "(a given app's prompt)" # Daemon._front: a given app's own system prompt is in front
369
+ PROMPT_RETURN_WAIT = 2.0 # seconds the app may take to come back once its prompt is answered
370
+ FRONT_RETRIES, FRONT_RETRY = 16, 0.5 # see Daemon._front: as long as a notification banner stays
371
+ MIN_VISIBLE = 10.0 # points: less of an element showing than this (high or wide) is no place to touch
372
+ # The status bar across the top of a phone held upright: a touch there scrolls the app's list to its top (iOS),
373
+ # whatever row lies under it. It is in no app's tree, so a row half under it looked reachable: a tap on Settings'
374
+ # "Notifications" row at y 10 scrolled the list instead (seen on the phone 2026-11-11). 50 pt covers the status
375
+ # bar of the current iPhones (47 to 54); in landscape an iPhone shows none.
376
+ STATUS_BAR = 50.0
377
+ MOVES_CONTENT = ("scroll", "swipe") # a drag: identical elements can't be told apart after these
378
+ # These may show the same screen again, somewhere else in it, or another: the elements say which. Calling each a
379
+ # move gave every element without an identifier a new ref, as a `+` and a `-` line for the same thing (seen on
380
+ # the phone 2026-11-10: open_url to the screen already open).
381
+ MAY_MOVE_CONTENT = ("back", "open_app", "open_url", "press_button", "rotate")
382
+ READS = ("look", "apps")
383
+ BUNDLE_ID = re.compile(r"^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+$")
384
+ SPECIAL = ("wait", "wait_for") # sleep before they answer, so they don't hold the phone's queue meanwhile
385
+ REQUIRED = {"rotate": ("to",), "open_app": ("app",), "close_app": ("app",), "toggle": ("target",), "wait": ("seconds",),
386
+ "type_text": ("text",), "tap": ("target",), "long_press": ("target",),
387
+ "swipe": ("x0", "y0", "x1", "y1"), "press_button": ("name",), "alert": ("action",),
388
+ "open_url": ("url",)}
389
+
390
+
391
+ async def _first(timeout: float, *events: asyncio.Event) -> None:
392
+ """Wait until any of the events is set, or `timeout` seconds."""
393
+ waits = [asyncio.create_task(e.wait()) for e in events]
394
+ try:
395
+ await asyncio.wait(waits, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
396
+ finally:
397
+ for w in waits:
398
+ w.cancel()
399
+
400
+
401
+ class Daemon:
402
+ def __init__(self, session, control: Controller | None = None, log=print, heartbeat: float = 5.0,
403
+ backoff: tuple = (1.0, 2.0, 5.0, 10.0), ready_wait: float = 60.0,
404
+ stream_linger: float = 30.0, stream_retry: float = 1.0, settle_still: float = STILL,
405
+ settle_first_frame: float = 1.0, settle_fallback: float = 0.6,
406
+ settle_quiet: float = MIN_QUIET, shot_wait: float = 1.0, shot_fresh: float = 1.0,
407
+ pace_tick: float | None = PACE_TICK, pace_quiet: tuple[float, float, float] = PACE_QUIET) -> None:
408
+ self.session = session
409
+ # the stream's pace (see STILL_FPS): how often it's checked (None: never), and the frame rate the phone
410
+ # streams at now (None until connected)
411
+ self.pace_tick, self.pace_quiet = pace_tick, pace_quiet
412
+ self._fps: int | None = None
413
+ self._person_at = self._changed_at = -math.inf # monotonic: a person's last input, the screen's last change
414
+ self._pace_seen: Frame | None = None # the newest frame at the last check
415
+ # screenshots: how long to wait for a stream frame before asking the phone, and how old one may be
416
+ self.shot_wait, self.shot_fresh = shot_wait, shot_fresh
417
+ self.unsettled = False # the last action returned without a settle: the screen may still be moving, for every client
418
+ # settle: how long a still screen is quiet, how long to wait for a first frame, and the
419
+ # blind sleep used when the stream gives no frames at all
420
+ self.settle_still, self.settle_first_frame, self.settle_fallback = settle_still, settle_first_frame, settle_fallback
421
+ self.settle_quiet = settle_quiet
422
+ self.control = control or Controller()
423
+ self.hub = FrameHub()
424
+ self.pump = StreamPump(self._open_stream, self.hub, linger=stream_linger, retry=stream_retry, log=log,
425
+ suspended=self._fed_by_video)
426
+ self.log = log
427
+ # A trial switch (TAPWRIGHT_AGENT_STREAM=0): tool calls don't open the JPEG stream, so an action settles
428
+ # on a short wait and a tree read. To learn whether the stream is what stalls requests over Wi-Fi.
429
+ self.agent_stream = os.environ.get("TAPWRIGHT_AGENT_STREAM", "1") != "0"
430
+ self._agent_touches = AgentTouches()
431
+ self._turned: tuple | None = None # ((frame seq, degrees), the turned JPEG): see turned()
432
+ self.heartbeat, self.backoff, self.ready_wait = heartbeat, backoff, ready_wait
433
+ self.state, self.detail = "connecting", None
434
+ self.preset = os.environ.get("TAPWRIGHT_PRESET", "balanced") # (a trial switch: the stream's preset at start)
435
+ if self.preset not in PRESETS:
436
+ self.preset = "balanced"
437
+ self.ws_clients = 0
438
+ self.video = None # a tapwright.video.VideoHub while the H.264 live view is set up (see start_video)
439
+ self.want_video = False # `tapwright serve --video`: set the video up once the phone is connected
440
+ # `--settle-on-video`: actions watch the video's frames and the JPEG stream stays closed. Off unless asked:
441
+ # a mirror that stalls sends nothing, exactly as a still screen does, so an action could answer "No
442
+ # visible change" for a screen that changed. Not for use until a check against the phone guards that.
443
+ self.settle_on_video = False
444
+ self._receiver = None # the AirPlay receiver process that feeds `video`
445
+ self._video_task: asyncio.Task | None = None
446
+ self._feeder: PictureFeeder | None = None # feeds `hub` from `video` while the hub is live (_start_feeder)
447
+ self._feeder_task: asyncio.Task | None = None
448
+ self.last_request = time.monotonic()
449
+ self._ever_connected = False
450
+ # why the last connect couldn't finish, when the phone said: a PhoneLocked, or an AppNotAnswering
451
+ self.stuck: PhoneLocked | AppNotAnswering | None = None
452
+ self._connected = asyncio.Event()
453
+ self._lost = asyncio.Event()
454
+ self._wake = asyncio.Event() # connect again now, without waiting out the backoff
455
+ self._connecting = False # a connect is under way
456
+ self._attempts = 0 # connects that ended, either way
457
+ self._scopes: dict[tuple[str, ...], dict[str, str]] = {} # each app list seen, as {bundle id: shown}
458
+ # {agent: the app its last touch near the top was meant for}, for a look: false answer that asked once
459
+ # {agent: (the app, when, the person generation then)}: see _recheck_later and _check_again
460
+ self._recheck: dict[str, tuple[tuple[str, str], float, int]] = {}
461
+ self._taken_notes: dict[str, str] = {} # {agent: a note _check_again found for its look}
462
+ self._queues: set = set()
463
+ self._handlers = {"open_app": self._open_app, "close_app": self._close_app, "toggle": self._toggle,
464
+ "tap": self._tap, "long_press": self._long_press, "back": self._back, "type_text": self._type_text, "scroll": self._scroll}
465
+
466
+ # ---- status and events ----
467
+ def status(self) -> dict:
468
+ w, h = self.session.screen
469
+ via = (getattr(self.session.device, "via", None) or "").lower()
470
+ via = ("usb" if via == "usb" else "wifi" if via == "local network"
471
+ else "tailscale" if via.startswith("network") else None)
472
+ return {"type": "status", "protocol": PROTOCOL, "tool": "Tapwright", "state": self.state, "detail": self.detail, "via": via,
473
+ "fps": self.hub.fps(), "driver": self.control.driver, "paused": self.control.paused,
474
+ "size": {"w": w, "h": h}, "quality": self.preset,
475
+ "accepts": [*ACCEPTS, "video"] if self.video is not None and self.video.live else ACCEPTS}
476
+
477
+ def subscribe(self) -> asyncio.Queue:
478
+ self._asked_for()
479
+ queue: asyncio.Queue = asyncio.Queue(maxsize=200)
480
+ self._queues.add(queue)
481
+ return queue
482
+
483
+ def unsubscribe(self, queue: asyncio.Queue) -> None:
484
+ self._queues.discard(queue)
485
+
486
+ def broadcast(self, message: dict) -> None:
487
+ """To every live view; one that can't keep up loses its oldest messages."""
488
+ for queue in list(self._queues):
489
+ if queue.full():
490
+ queue.get_nowait()
491
+ queue.put_nowait(message)
492
+
493
+ def _set_state(self, state: str, detail: str | None = None) -> None:
494
+ changed = (state, detail) != (self.state, self.detail)
495
+ self.state, self.detail = state, detail
496
+ if state in ("live", "locked"):
497
+ self._connected.set()
498
+ else:
499
+ self._connected.clear()
500
+ if changed:
501
+ self.broadcast(self.status())
502
+
503
+ def _point(self, point) -> dict | None:
504
+ if not point:
505
+ return None
506
+ w, h = self.session.screen
507
+ return {"x": round(point[0] / w, 4), "y": round(point[1] / h, 4)}
508
+
509
+ def _announce(self, by: str, kind: str, label: str | None = None, point=None, ok: bool = True,
510
+ code: str | None = None, caption: str | None = None, reason: str | None = None) -> None:
511
+ message = {"type": "action", "by": by, "kind": kind, "at": int(time.time() * 1000), "ok": ok}
512
+ if label:
513
+ message["label"] = label
514
+ if caption:
515
+ message["caption"] = caption
516
+ if reason:
517
+ message["reason"] = reason
518
+ if point:
519
+ message["point"] = point
520
+ if code:
521
+ message["code"] = code
522
+ self.broadcast(message)
523
+
524
+ # ---- the connection ----
525
+ def lost(self, reason: "str | Exception") -> None:
526
+ """A request found the line dead: the supervisor reconnects. When the request was cut because the app in
527
+ front doesn't answer (AppNotAnswering), that is what agents and the live view are told meanwhile."""
528
+ stuck = reason if isinstance(reason, AppNotAnswering) else None
529
+ if stuck is not None:
530
+ self.stuck = stuck
531
+ if self.state in ("live", "locked"):
532
+ self.log(f"connection lost: {reason}")
533
+ self._set_state("not-answering" if stuck is not None else "reconnecting", str(reason)[:200])
534
+ for view in self.control.clients.values():
535
+ view.dropped = True # every agent's list is from before the drop
536
+ self._lost.set()
537
+
538
+ def _not_connected(self) -> Refused:
539
+ """No phone found or not set up: the agent can't fix that by waiting. Otherwise the link
540
+ is coming back."""
541
+ if self.state == "no-device":
542
+ return Refused("not_connected", f"no phone to drive ({self.detail}); fix that, then try again")
543
+ if isinstance(self.stuck, PasscodeNeeded):
544
+ return Refused("locked", "the phone is asking for its passcode to allow UI automation; ask the user "
545
+ "to enter it on the phone")
546
+ if isinstance(self.stuck, PhoneLocked):
547
+ return Refused("locked", "the phone is locked; ask the user to unlock it")
548
+ if isinstance(self.stuck, AppNotAnswering):
549
+ return Refused("not_answering", f"{self.stuck}. close_app can still close it; then open_app it again.")
550
+ return Refused("reconnecting", f"not connected to the phone ({self.detail or self.state}); "
551
+ "wait a few seconds, then look")
552
+
553
+ def _require_connected(self) -> None:
554
+ if not self._connected.is_set():
555
+ raise self._not_connected()
556
+
557
+ async def _connected_only(self, fn):
558
+ """Run `fn` only if the line is up when its turn comes: a reconnect holds the queue,
559
+ and whatever waited for it must not reconnect on its own."""
560
+ self._require_connected()
561
+ return await fn()
562
+
563
+ def _asked_for(self) -> bool:
564
+ """A tool call or a viewer wants the phone while it waits behind the passcode screen: put that screen up
565
+ now if it has gone (the starts thin out when nobody asks, see device.PROMPT_PAUSES). True when a start
566
+ was asked for."""
567
+ if isinstance(self.stuck, PasscodeNeeded):
568
+ hurry = getattr(self.session.device, "hurry", None)
569
+ if hurry is not None:
570
+ hurry()
571
+ self._wake.set()
572
+ return True
573
+ return False
574
+
575
+ async def _next_start(self) -> None:
576
+ """Wait for a connect that began after now to end (or for the line to come up). What the last start
577
+ found may be minutes old: the person may have typed the passcode since, with nothing started to notice
578
+ (seen on the phone 2026-11-10: a tap answered `locked` 20 s before the phone was back). A connect
579
+ already under way may have looked before it was asked to hurry, so the one after it counts."""
580
+ need = self._attempts + (2 if self._connecting else 1)
581
+ deadline = time.monotonic() + self.ready_wait
582
+ while self._attempts < need and not self._connected.is_set() and time.monotonic() < deadline:
583
+ await asyncio.sleep(0.05)
584
+
585
+ async def wait_connected(self) -> None:
586
+ if self._asked_for():
587
+ await self._next_start()
588
+ if self.state == "no-device" or self.stuck is not None: # waiting won't fix either
589
+ raise self._not_connected()
590
+ try:
591
+ await asyncio.wait_for(self._connected.wait(), self.ready_wait)
592
+ except asyncio.TimeoutError:
593
+ raise self._not_connected() from None
594
+
595
+ async def _connect(self) -> None:
596
+ """A working WebDriverAgent session: a new WebDriverAgent if the tunnel still works,
597
+ otherwise everything from scratch."""
598
+ async with self.control.exclusive():
599
+ s = self.session
600
+ if s.device.rsd is not None:
601
+ s.sid = None
602
+ try:
603
+ await s.ensure()
604
+ except (NotSetUp, PhoneLocked, AppNotAnswering): # the tunnel is fine: rebuilding it won't help
605
+ raise
606
+ except Exception as e:
607
+ self.log(f"restarting WebDriverAgent didn't work ({e}); reconnecting")
608
+ await s.reset()
609
+ await s.ensure()
610
+ else:
611
+ await s.ensure()
612
+ await s.settings({**PRESETS[self.preset], **NO_IDLE_WAIT})
613
+ self._fps = self._preset_fps()
614
+ await s.read_scale(refresh=True) # once per connection, so every header says @3x, not only SpringBoard's
615
+ await s.read_app_names(refresh=True) # so the header's App: line says "Settings", not a bundle id
616
+
617
+ async def supervise(self, stop: asyncio.Event) -> None:
618
+ """Connect, then keep checking; reconnect whenever the line drops. Runs until `stop`."""
619
+ self._agent_touches.start() # find Tapline's hosts, if any, before the first touch needs them
620
+ pacer = asyncio.create_task(self._pace(stop), name="pace") if self.pace_tick else None
621
+ watch = asyncio.create_task(self._watch_loop(stop), name="loop-watch") if self.pace_tick else None
622
+ try:
623
+ await self._supervise(stop)
624
+ finally:
625
+ for helper in (pacer, watch):
626
+ if helper is not None:
627
+ helper.cancel()
628
+ await asyncio.gather(helper, return_exceptions=True)
629
+
630
+ async def _watch_loop(self, stop: asyncio.Event) -> None:
631
+ """Say in the log when this process's event loop was held up. The tunnel to the phone is a TCP/IP stack
632
+ in pure Python on this same loop: while something holds the loop, no packet moves, and a held loop looks
633
+ from outside like a phone that doesn't answer."""
634
+ loop = asyncio.get_running_loop()
635
+ worst, told = 0.0, loop.time()
636
+ while not stop.is_set():
637
+ before = loop.time()
638
+ await asyncio.sleep(LOOP_TICK)
639
+ worst = max(worst, loop.time() - before - LOOP_TICK)
640
+ if worst >= LOOP_HELD and loop.time() - told >= LOOP_SAY_EVERY:
641
+ self.log(f"the event loop was held for {worst:.2f} s; the tunnel's packets waited meanwhile")
642
+ worst, told = 0.0, loop.time()
643
+
644
+ # ---- the stream's pace ----
645
+ def _preset_fps(self) -> int:
646
+ return int(PRESETS[self.preset]["mjpegServerFramerate"])
647
+
648
+ async def _set_fps(self, fps: int) -> None:
649
+ await self.session.settings({"mjpegServerFramerate": fps})
650
+ self._fps = fps
651
+
652
+ async def _quicken(self) -> None:
653
+ """Back to the preset's rate before an action, so its settle (and the person's picture) has every frame.
654
+ Only after a quiet spell; a failure to change the rate never stops the action."""
655
+ if self._fps is not None and self._fps < self._preset_fps() and not self._fed_by_video():
656
+ try:
657
+ await self._set_fps(self._preset_fps())
658
+ except ConnectionLost:
659
+ raise
660
+ except DeviceError as e:
661
+ self.log(f"pace: couldn't raise the frame rate: {e}")
662
+
663
+ def _fed_by_video(self) -> bool:
664
+ """Is the AirPlay video the hub's source now (so the MJPEG stream stays closed)?"""
665
+ return self._feeder is not None and self._feeder.feeding
666
+
667
+ async def _pace(self, stop: asyncio.Event) -> None:
668
+ while not stop.is_set():
669
+ await _first(self.pace_tick, stop)
670
+ if stop.is_set():
671
+ return
672
+ try:
673
+ await self._pace_once()
674
+ except ConnectionLost as e:
675
+ self.lost(e)
676
+ except Refused:
677
+ pass # not connected, or paused: the next check tries again
678
+ except Exception as e:
679
+ self.log(f"pace: {e}")
680
+
681
+ async def _pace_once(self) -> None:
682
+ """Slow the stream when nothing has happened for a while, and quicken it when the screen changes."""
683
+ if self._fed_by_video():
684
+ return # the MJPEG stream is closed: there is no frame rate to set
685
+ now = time.monotonic()
686
+ frame = self.hub.latest
687
+ if frame is not None and frame is not self._pace_seen:
688
+ if not alike(frame, self._pace_seen): # a first frame counts as a change: nothing says it's still yet
689
+ self._changed_at = now
690
+ self._pace_seen = frame
691
+ if self.state != "live" or self._fps is None or frame is None or self.hub.fps() == 0:
692
+ return # nothing streams, so the pace costs nothing
693
+ agent, person, still = self.pace_quiet
694
+ busy = (now - self.last_request < agent or now - self._person_at < person or now - self._changed_at < still)
695
+ want = self._preset_fps() if busy else min(STILL_FPS, self._preset_fps())
696
+ if want != self._fps:
697
+ await self.control.read(lambda: self._connected_only(lambda: self._set_fps(want)))
698
+
699
+ async def _supervise(self, stop: asyncio.Event) -> None:
700
+ attempt = failures = 0
701
+ while not stop.is_set():
702
+ if self.state not in ("live", "locked"):
703
+ self._connecting = True
704
+ try:
705
+ try:
706
+ await self._connect()
707
+ finally:
708
+ self._connecting = False
709
+ self._attempts += 1
710
+ except NotSetUp as e:
711
+ self._set_state("no-device", f"needs setup: {e}")
712
+ except PhoneNotFound:
713
+ self._set_state("no-device", "phone not found: plug it in and unlock it")
714
+ except (PhoneLocked, AppNotAnswering) as e:
715
+ if self.stuck is None:
716
+ self.log(f"connected, but {e}")
717
+ self.stuck = e
718
+ state = "reconnecting" if self._ever_connected else "connecting"
719
+ self._set_state("not-answering" if isinstance(e, AppNotAnswering) else state, str(e)[:200])
720
+ except Exception as e:
721
+ self.stuck = None
722
+ self._set_state("reconnecting" if self._ever_connected else "connecting", str(e)[:200])
723
+ else:
724
+ attempt = failures = 0
725
+ self.stuck = None
726
+ self._ever_connected = True
727
+ self._lost.clear() # a drop noticed while reconnecting is already dealt with
728
+ self._set_state("live")
729
+ self.log(f"connected over {self.session.device.via}")
730
+ if self.want_video and self._video_task is None:
731
+ self._video_task = asyncio.create_task(self.start_video(), name="video")
732
+ continue
733
+ await _first(self.backoff[min(attempt, len(self.backoff) - 1)], stop, self._wake)
734
+ self._wake.clear()
735
+ attempt += 1
736
+ continue
737
+ await _first(self.heartbeat, stop, self._lost)
738
+ if stop.is_set():
739
+ break
740
+ if self._lost.is_set():
741
+ self._lost.clear()
742
+ if self.state in ("live", "locked"):
743
+ self._set_state("reconnecting", "connection lost")
744
+ continue
745
+ try:
746
+ locked = await self.control.read(lambda: self._connected_only(self.session.locked))
747
+ except ConnectionLost as e:
748
+ self.lost(e)
749
+ except Refused:
750
+ pass
751
+ except Exception as e:
752
+ failures += 1
753
+ self.log(f"heartbeat: {e}")
754
+ if failures >= 3:
755
+ failures = 0
756
+ self.lost(f"WebDriverAgent keeps failing: {e}")
757
+ else:
758
+ failures = 0
759
+ if not self._lost.is_set() and self.state in ("live", "locked"):
760
+ if locked:
761
+ self._set_state("locked", "unlock the phone")
762
+ else:
763
+ self._set_state("live")
764
+
765
+ # ---- agent tools ----
766
+ async def run_tool(self, client_id: str, name: str, args: dict, scope: "Scope | None" = None) -> dict:
767
+ """One agent tool call: {"text": str, "image": base64 JPEG or None, "image_size": [w, h] or None}.
768
+ Raises Refused with a code from errors.CODES; any other failure is `internal` (logged).
769
+ `scope` is the apps this agent was given (personal mode); None is a dedicated device: no limit."""
770
+ self.last_request = time.monotonic()
771
+ try:
772
+ # The stream is held open for the length of every tool call; the pump's own linger keeps
773
+ # it open for 30 s after the last one, so the settle always has frames to read.
774
+ # Not when there is no phone: the pump would only retry opening it.
775
+ async with self.pump.watching() if self._connected.is_set() and self.agent_stream else nullcontext():
776
+ await self._check_again(client_id, name) # before personal mode's front check: see there
777
+ if scope is None:
778
+ return await self._run_tool(client_id, name, args)
779
+ return await self._within(scope, client_id, name, args)
780
+ except Refused:
781
+ raise
782
+ except DeviceError as e: # WebDriverAgent or the link answered with an error; not our bug
783
+ self.log(f"{name}: the phone side failed: {e}")
784
+ raise Refused("internal", f"the phone side failed: {e}") from e
785
+ except Exception as e: # one failed step never takes the daemon down
786
+ self.log(f"{name} failed unexpectedly:\n{traceback.format_exc()}")
787
+ raise Refused("internal", f"{name} failed unexpectedly: {e}. This is a bug; report it.") from e
788
+
789
+ # ---- personal mode: the agent stays in the apps it was given ----
790
+ async def _given(self, scope: "Scope") -> dict[str, str]:
791
+ """{bundle id: how to show it} of the apps in a scope. An entry that looks like a bundle id is taken as
792
+ it is; a name is looked up among the installed apps, once."""
793
+ if scope.apps not in self._scopes:
794
+ given: dict[str, str] = {}
795
+ for entry in scope.apps:
796
+ if BUNDLE_ID.match(entry):
797
+ given[entry] = entry
798
+ continue
799
+ await self.wait_connected()
800
+ try:
801
+ bundle, shown = await self._app(entry, partial=False)
802
+ except Refused as e:
803
+ raise Refused("unsupported", f'the app list this agent was started with names "{entry}", which '
804
+ f"can't be used: {e}") from None
805
+ given[bundle] = shown
806
+ self._scopes[scope.apps] = given
807
+ return self._scopes[scope.apps]
808
+
809
+ async def _front(self, given: dict[str, str], prompt=None) -> str | None:
810
+ """The app in front, for personal mode's question before a touch. WebDriverAgent answers SpringBoard
811
+ while a notification banner slides over the app in front (seen on the phone 2026-11-10: twice in forty
812
+ calls, each a false `outside_apps`; a banner stays for some seconds). When it says SpringBoard and the phone's
813
+ own process list has one of the given apps in the foreground, it is asked again, FRONT_RETRIES times
814
+ FRONT_RETRY s apart at most. Only WebDriverAgent's answer lets a call through; the process list never
815
+ does, and on the real home screen it has no given app, so that refusal isn't delayed. `prompt`: asked first
816
+ whether the screen is a given app's own permission prompt (see _prompt_of_given): then PROMPT_FRONT at once,
817
+ with no wait, since a prompt doesn't leave by itself as a banner does."""
818
+ front = await self._read(self.session.foreground_bundle_id)
819
+ if front != SPRINGBOARD:
820
+ return front
821
+ try:
822
+ running = set(await asyncio.wait_for(self.session.device.foreground_apps(), 5))
823
+ except Exception: # no process list in time: WebDriverAgent's word stands
824
+ return front
825
+ if running & set(given) and prompt is not None and await prompt():
826
+ return PROMPT_FRONT
827
+ for _ in range(FRONT_RETRIES if running & set(given) else 0):
828
+ await asyncio.sleep(FRONT_RETRY)
829
+ front = await self._read(self.session.foreground_bundle_id)
830
+ if front != SPRINGBOARD:
831
+ break
832
+ return front
833
+
834
+ async def _prompt_of_given(self, given: dict[str, str], view, read: bool) -> bool:
835
+ """Is the system alert on screen a given app's own: a permission prompt the app raised ("“Expo Go” Would
836
+ Like to Send You Notifications")? iOS draws it in SpringBoard, so personal mode saw the home screen in front
837
+ and refused the very `alert` the skill says to answer (a first-time agent stuck on it, 2026-11-11). Only
838
+ when the alert's title names a given app and the phone's process list still has that app in the
839
+ foreground under it. `read`: read the screen first (a touch's check); else the answer's own read counts."""
840
+ if read:
841
+ await self._read(lambda: self._silent_read(view))
842
+ if view.alert_now is None:
843
+ return False
844
+ title = view.alert_now[0]
845
+ named = [b for b in given if (self.session.app_names.get(b) or given[b].split(" (")[0]) in title]
846
+ if not named:
847
+ return False
848
+ try:
849
+ running = set(await asyncio.wait_for(self.session.device.foreground_apps(), 5))
850
+ except Exception:
851
+ return False # the phone can't say: refuse, the safe side
852
+ return bool(running & set(named))
853
+
854
+ async def _back_from_prompt(self, client_id: str, given: dict[str, str], answer: dict) -> dict:
855
+ """After a given app's own permission prompt is answered, the app comes back as the prompt fades. The
856
+ answer's read can come mid-fade and show SpringBoard, which personal mode withholds; a first-time agent then
857
+ reopened the app for nothing (2026-11-11). So wait for the app, up to PROMPT_RETURN_WAIT s, and answer
858
+ with its screen."""
859
+ shown = APP_LINE.search(answer["text"]) # as the check below decides: the App: line, else the last read's app
860
+ app = shown.group(2) if shown else (self.session.app or (None, None))[1]
861
+ if app in given:
862
+ return answer
863
+ loop = asyncio.get_running_loop()
864
+ until = loop.time() + PROMPT_RETURN_WAIT
865
+ while await self._read(self.session.foreground_bundle_id) not in given:
866
+ if loop.time() >= until:
867
+ return answer
868
+ await asyncio.sleep(0.25)
869
+ fresh = await self._run_tool(client_id, "look", {"screenshot": False})
870
+ return {**fresh, "text": answer["text"].splitlines()[0] + "\n" + fresh["text"]}
871
+
872
+ async def _within(self, scope: "Scope", client_id: str, name: str, args: dict) -> dict:
873
+ """One tool call in personal mode, failing closed. Nothing runs without an app list. Opening or closing an
874
+ app outside the list is refused. Reading or touching is refused while another app is in front (asked of
875
+ the phone before the call), and an answer never shows another app's screen."""
876
+ if not scope.apps:
877
+ raise Refused("not_allowed", NO_APPS)
878
+ given = await self._given(scope)
879
+ names = ", ".join(given.values())
880
+ front = None
881
+ if name in ("open_app", "close_app") and isinstance(args.get("app"), str) and args["app"].strip():
882
+ if name == "close_app" and isinstance(self.stuck, AppNotAnswering):
883
+ bundle, shown = self.stuck.bundle_id, f"{self.stuck.name} ({self.stuck.bundle_id})"
884
+ else:
885
+ await self.wait_connected()
886
+ bundle, shown = await self._app(args["app"], partial=name == "open_app")
887
+ if bundle not in given:
888
+ raise Refused("not_allowed", f"outside_apps: {shown} isn't one of the apps you were given ({names}).")
889
+ elif name in ACTIONS and name not in SCOPE_FREE or name == "wait_for":
890
+ # A look needs no question first: its own read says which app is in front, and an answer from
891
+ # another app is withheld below. Touches, and wait_for (it reveals whether a text is there), ask.
892
+ await self.wait_connected()
893
+ view = self.control.client(client_id)
894
+ front = await self._front(given, prompt=lambda: self._prompt_of_given(given, view, read=True))
895
+ if front not in given and front != PROMPT_FRONT:
896
+ raise Refused("not_allowed", _outside(front, self.session.app_names.get(front or ""), names))
897
+ answer = await self._run_tool(client_id, name, args)
898
+ if name == "alert" and front == PROMPT_FRONT:
899
+ answer = await self._back_from_prompt(client_id, given, answer)
900
+ if name == "apps": # only its own apps: the rest of the phone is the owner's business
901
+ lines = answer["text"].splitlines()
902
+ kept = [ln for ln in lines if ln.split(" ", 1)[0] in given or not BUNDLE_ID.match(ln.split(" ", 1)[0])]
903
+ return {**answer, "text": "\n".join(kept)}
904
+ # The answer's own App: line says whose screen it lists; without one, the app at the last read does.
905
+ shown = APP_LINE.search(answer["text"])
906
+ app = (shown.group(1), shown.group(2)) if shown else self.session.app
907
+ if (app is not None and app[1] not in given and ("\n" in answer["text"] or answer.get("image"))
908
+ and not (app[1] == SPRINGBOARD
909
+ and await self._prompt_of_given(given, self.control.client(client_id), read=True))):
910
+ if name == "look": # nothing was done, so there is nothing to report but the refusal
911
+ raise Refused("not_allowed", _outside(app[1], app[0], names))
912
+ first = answer["text"].splitlines()[0]
913
+ first = "" if first.startswith("Screen ") else first + "\n" # a look's first line is the screen itself
914
+ return {"text": f"{first}Note: {_outside(app[1], app[0], names)}", "image": None, "image_size": None}
915
+ return answer
916
+
917
+ async def _run_tool(self, client_id: str, name: str, args: dict) -> dict:
918
+ if name not in ACTIONS and name not in READS and name not in SPECIAL:
919
+ raise Refused("unsupported", f"no tool named {name!r}; the tools are "
920
+ f"{', '.join([*READS, *SPECIAL, *ACTIONS])}")
921
+ missing = [k for k in REQUIRED.get(name, ()) if args.get(k) is None]
922
+ if missing:
923
+ raise Refused("bad_params", f"{name} needs {', '.join(missing)}")
924
+ args = _checked(name, args)
925
+ shot = self._shot_args(args, default=name == "look", custom=name == "look") # bad width: refused here
926
+ # A bad target was refused by _checked, before anything is queued or sent. Any tool whose
927
+ # arguments carry `target` goes through here, so the live-read rule in act() covers it too.
928
+ target = args.get("target")
929
+ view = self.control.client(client_id)
930
+ # an agent that hasn't looked yet has no "last look": its first list is whole anyway
931
+ notes = [NOTE_PERSON] if 0 <= view.seen < self.control.person_generation else []
932
+ if name == "close_app" and isinstance(self.stuck, AppNotAnswering):
933
+ return await self._close_stuck(self.stuck, args["app"], notes)
934
+ await self.wait_connected()
935
+ if (taken := self._taken_notes.pop(client_id, None)) is not None:
936
+ notes = [*notes, taken]
937
+ if name == "apps":
938
+ return await self._apps(notes)
939
+ if name == "wait":
940
+ return await self._wait(view, args, notes, shot)
941
+ if name == "wait_for":
942
+ return await self._wait_for(view, args, notes, shot)
943
+ if name == "look":
944
+ await self._settle_pending()
945
+ return await self._look(view, shot, [], notes)
946
+ if self.state == "locked":
947
+ raise Refused("locked", "the phone is locked; ask the user to unlock it")
948
+ call, settle = ACTIONS[name]
949
+ if session_module.TRACE:
950
+ self.log(f"trace {name} begins")
951
+ by_ref = target is not None and target.kind == "ref"
952
+ # these tools at a point act on the element under it, so they need a current list too
953
+ by_lookup = target is not None and (target.kind in ("id", "label", "text")
954
+ or (name in ("type_text", "toggle", "scroll") and target.kind == "point"))
955
+
956
+ fresh, full, box = [True], [False], {}
957
+
958
+ async def act():
959
+ self._require_connected()
960
+ await self._quicken()
961
+ s = self.session
962
+ # a list read mid-redraw can miss elements while the screen looks unchanged since: a target missing
963
+ # from it is read for again before it is not_found
964
+ live = by_lookup and (not self._current(view) or (target.kind != "point" and self._missing(view.items, target)))
965
+ if not live: # a silent re-read settles for itself
966
+ await self._settle_pending()
967
+ s.items = view.items
968
+ fresh[0] = view.seen == self.control.generation
969
+ # never looked, a person acted, or the line dropped: the agent's list may be anything
970
+ full[0] = not fresh[0] or view.dropped or view.read_at == float("-inf")
971
+ if by_ref and view.dropped:
972
+ raise Refused("stale", "the connection to the phone dropped since your last look; look again")
973
+ if by_ref and not fresh[0]:
974
+ raise Refused("stale", "the screen changed since your last look (someone else used the "
975
+ "phone); look again")
976
+ if live:
977
+ # Resolve on the live tree. The agent's refs follow their elements onto it, so `e3` still
978
+ # means the same element when it moved. The view's freshness doesn't change: a person's
979
+ # action still makes refs stale, and `view.shown` (what the agent was told) stays put.
980
+ if await self._silent_read(view) and self._missing(s.items, target):
981
+ # the app may have been mid-redraw: one more look, once it is still, before saying so
982
+ await self._silent_read(view)
983
+ if by_lookup and target.kind != "point" and self._missing(s.items, target):
984
+ if self._missing(s.items, dataclasses.replace(target, index=None)):
985
+ # an alert hides the app's elements: say so, not "not found". (When only the `index` is
986
+ # out of range the alert's own elements matched, and that miss is the answer.)
987
+ self._refuse_behind_alert(view, force=False)
988
+ self._refuse_behind_sheet(target) # before off_screen: scrolling can't help until the sheet is closed
989
+ if not (name == "scroll" and args.get("to") == "target"): # that scroll is how it comes on screen
990
+ if name in KEYBOARD_TOOLS and view.read_keyboard and self._under_keyboard(target):
991
+ # the keyboard hides it, and a short form can't scroll it up: a person closes the keyboard
992
+ # first (a first-time agent was told off_screen for Wayfare's "Create account", 2026-11-11)
993
+ closed = await self._dismiss_keyboard_now(view)
994
+ if not self._missing(s.items, target):
995
+ box["closed"] = closed
996
+ if "closed" not in box:
997
+ self._refuse_off_screen(target)
998
+ # a person would close the keyboard before the touch
999
+ box["note"] = box.pop("closed", "") or await self._close_keyboard(view, name, target)
1000
+ s.last_point = None
1001
+ box["app"], s.top_touch = s.app, False # the app the touch is meant for (see _taken_away)
1002
+ box["mark"] = self._mark() # where the stream stands as the gesture goes out
1003
+ handler = self._handlers.get(name)
1004
+ return await (handler(view, args) if handler else call(s, args))
1005
+
1006
+ async def act_as_agent():
1007
+ # the touches inside are the agent's: Tapline is told before each one (tapline.py). Set only here, so a
1008
+ # person's input from the live view, queued separately, is never announced as the agent's
1009
+ self.session.touch_hook, self.session.banner_guard = self._agent_touches.notify, True
1010
+ try:
1011
+ return await act()
1012
+ finally:
1013
+ self.session.touch_hook, self.session.banner_guard = None, False
1014
+
1015
+ try:
1016
+ result = await self.control.agent(act_as_agent)
1017
+ except Unanswered as e:
1018
+ # the line is alive and the session stays; the screen may or may not have changed under the agent's list
1019
+ view.moved = True
1020
+ self.unsettled = True
1021
+ self._announce("agent", name, ok=False, code="timeout")
1022
+ raise Refused("timeout", f"{e}; the action may have happened: look before you retry.") from None
1023
+ except ConnectionLost as e:
1024
+ self.lost(e)
1025
+ if isinstance(e, AppNotAnswering): # the action went out, then the app stopped answering
1026
+ self._announce("agent", name, ok=False, code="not_answering")
1027
+ raise self._not_connected() from None
1028
+ self._announce("agent", name, ok=False, code="reconnecting")
1029
+ raise Refused("reconnecting", "the connection dropped during the action; look to check what "
1030
+ "happened") from None
1031
+ except Refused as e:
1032
+ # a touch a finger couldn't make: people watching see the step and why (VIEWER-PROTOCOL 1.4 `reason`)
1033
+ if e.code == "not_actionable":
1034
+ said = str(e)
1035
+ word = said.split(":", 1)[0] if ":" in said else ""
1036
+ self._announce("agent", name, label=element_name(said) or None, ok=False, code=e.code,
1037
+ reason=word if word.isidentifier() else None, caption=step_caption(name, said, args))
1038
+ raise
1039
+ if isinstance(result, Noted) and result.note:
1040
+ notes = [*notes, result.note]
1041
+ # The keyboard note is given only when the keyboard is still closed as the call answers (AGENT-TOOLS
1042
+ # v1.4): type_text focuses its input, so the keyboard is up again and the note would contradict the header
1043
+ closed_first = box.get("note") if name != "type_text" else None
1044
+ if closed_first:
1045
+ notes = [*notes, closed_first]
1046
+ if fresh[0]: # its own action keeps what it knew current; after someone else's, it must look
1047
+ view.seen = self.control.generation
1048
+ if isinstance(result, Unchanged): # nothing was done: nothing to settle or diff
1049
+ return _answer([str(result)], notes)
1050
+ reads_only = name == "alert" and args.get("action") == "text" # it changes nothing on the phone
1051
+ looks = settle is not None and args.get("look", True) and not reads_only
1052
+ view.moved = view.moved or not looks
1053
+ if not reads_only:
1054
+ self.unsettled = not looks
1055
+ self._announce("agent", name, label=element_name(str(result)), caption=step_caption(name, str(result), args),
1056
+ point=self._point(self.session.last_point))
1057
+ if not looks:
1058
+ view.unlooked += 0 if reads_only else 1
1059
+ taken = await self._taken_away(view, box.get("app"))
1060
+ self._recheck_later(client_id, box.get("app"), taken)
1061
+ return _answer([str(result)], [*notes, taken] if taken else notes)
1062
+ mark = box["mark"] # the stream as the gesture went out; the settle only trusts frames after the return
1063
+ # frames during the call count for actions with the normal cap; a launch may show a still splash first
1064
+ if isinstance(result, Toggled):
1065
+ landed = await self._toggle_landed(view, result)
1066
+ elif isinstance(result, Typed) and args.get("replace", True) and not args.get("submit"):
1067
+ landed = await self._typed_landed(view, result, args["text"])
1068
+ if landed is None and not result.field.secure and await self._typed_wrong(result, args["text"]) and \
1069
+ await self.control.agent(lambda: self.session.retype(args["text"])):
1070
+ # the field doesn't hold what was typed: keys went missing (iOS swapping in another keyboard as
1071
+ # the field took focus, seen on the phone 2026-11-11: "abc" landed as "a", and the retry with the
1072
+ # caret at the start made it "abca"). Cleared and typed once more, and the answer says so.
1073
+ notes = [*notes, NOTE_RETYPED]
1074
+ landed = await self._typed_landed(view, result, args["text"])
1075
+ else:
1076
+ landed = None
1077
+ if landed is not None:
1078
+ done = Settled(landed[1], True, False)
1079
+ else:
1080
+ done = await self._settle(self._mark()[0], cap=settle, since=mark[1],
1081
+ early=mark[0] if settle == CAP else None)
1082
+ if session_module.TRACE:
1083
+ self.log(f"trace {name} settle {done.seconds:.2f} s, changed={done.changed}")
1084
+ if done.capped:
1085
+ notes = [*notes, NOTE_CAPPED.format(cap=settle)]
1086
+ if name == "back" and not done.changed:
1087
+ raise Refused("not_actionable", "no_action: nothing changed after back" + self._still_open())
1088
+ # a toggle that changed nothing on the stream is not trusted: the tree says whether the switch flipped
1089
+ # Never on the AirPlay video's word alone: a mirror that stalls repeats its last picture, which looks exactly
1090
+ # like a still screen, so "No visible change" could be false. Then the tree read decides (0.16 s).
1091
+ if (not done.changed and name != "toggle" and not (full[0] or view.moved) and not view.shown_off
1092
+ and not self._fed_by_video()
1093
+ and (mark[1] is view.read_mark[1] or alike(mark[1], view.read_mark[1]))):
1094
+ # no frame changed after the call returned, and the picture at its start is the one the agent's
1095
+ # list was read against (nothing changed silently before it): no tree read, the list stays.
1096
+ # Not when something lies off screen: the action's effect may be there, and only a read shows it
1097
+ view.read_at, view.read_mark = self.control.clock(), self._mark()
1098
+ view.unlooked = 0 # this block covers them: nothing changed since the list
1099
+ return await self._add_shot(_answer([str(result), changes.SAME_SCREEN, changes.NO_CHANGE,
1100
+ settled_line(done.seconds)], notes), view, shot, at_read=False)
1101
+ to_target = name == "scroll" and args.get("to") == "target"
1102
+ # (a scroll to a target read the list after each of its drags, with the refs carried: its last read is no
1103
+ # move for the refs, though the content did move since the agent's last answer)
1104
+ moves = None if name in MAY_MOVE_CONTENT else name in MOVES_CONTENT and not to_target
1105
+ answer = await self._after(view, str(result), notes, full[0], moves, done.seconds, drifted=to_target,
1106
+ snap=landed[0] if landed is not None else None)
1107
+ if closed_first and view.read_keyboard: # the tap focused another input: the keyboard is up again
1108
+ answer["text"] = answer["text"].replace(f"\nNote: {closed_first}", "", 1)
1109
+ if isinstance(result, Toggled):
1110
+ # say only what the new list shows; if the switch can't be found again, claim nothing
1111
+ old = result.before
1112
+ same = [e for e in view.items if (e.role, e.label, e.identifier) == (old.role, old.label, old.identifier)]
1113
+ # by its ref; when the answer is a full list that numbered it anew (an agent's first call), by being the
1114
+ # only switch with that identity
1115
+ now = next((e for e in same if e.ref == result.ref), same[0] if len(same) == 1 else None)
1116
+ if now is not None:
1117
+ answer["text"] = answer["text"].replace(str(result), result.verdict(now, bool(view.alert)), 1)
1118
+ if isinstance(result, Typed): # the value as the new list shows it, not as it was typed
1119
+ old = result.field
1120
+ alike_fields = [e for e in view.items if e.role == old.role and (
1121
+ e.identifier == old.identifier if old.identifier else e.label == old.label)]
1122
+ now = next((e for e in view.items if result.same_field(e)),
1123
+ alike_fields[0] if len(alike_fields) == 1 else None) # renumbered in a first full list
1124
+ if now is not None:
1125
+ answer["text"] = answer["text"].replace(str(result), result.verdict(now), 1)
1126
+ taken = await self._taken_away(view, box.get("app"), read=self.session.app)
1127
+ self._recheck_later(client_id, box.get("app"), taken)
1128
+ if taken: # the screen the agent is shown is the app's, back in front, not the other app's
1129
+ return await self._look(view, shot, [str(result)], [*notes, taken])
1130
+ return await self._add_shot(answer, view, shot)
1131
+
1132
+ def _recheck_later(self, client_id: str, before: tuple[str, str] | None, taken: str | None) -> None:
1133
+ """After a touch near the top that nothing was seen taking: the other app may come to the front a moment
1134
+ after the answer's check (seen on the phone 2026-11-11 with look: false), so the agent's next call asks
1135
+ again, if it comes within RECHECK_WITHIN and no person acted in between (Tapline's review)."""
1136
+ if taken is None and before is not None and self.session.top_touch:
1137
+ self._recheck[client_id] = (before, time.monotonic(), self.control.person_generation)
1138
+ else:
1139
+ self._recheck.pop(client_id, None)
1140
+
1141
+ async def _check_again(self, client_id: str, name: str) -> None:
1142
+ """At an agent's call: the check _recheck_later left for it. Done before personal mode's own check of the
1143
+ app in front, so the agent hears that a banner took its touch rather than `not_allowed`. An action is
1144
+ refused `stale` (its refs are for the app, back in front now); a look carries the note. `apps` reads
1145
+ nothing of the screen and leaves the check for the next call; open_app makes it moot."""
1146
+ if name == "apps" or (pending := self._recheck.pop(client_id, None)) is None or name == "open_app":
1147
+ return
1148
+ before, when, persons = pending
1149
+ if time.monotonic() - when > RECHECK_WITHIN or self.control.person_generation != persons:
1150
+ return
1151
+ await self.wait_connected()
1152
+ view = self.control.client(client_id)
1153
+ taken = await self._taken_away(view, before, top=True)
1154
+ if taken is None:
1155
+ return
1156
+ if name in ACTIONS:
1157
+ view.moved = True
1158
+ raise Refused("stale", f"after your last action: {taken} This call did nothing.")
1159
+ self._taken_notes[client_id] = taken
1160
+
1161
+ async def _taken_away(self, view, before: tuple[str, str] | None,
1162
+ read: tuple[str, str] | None = None, top: bool | None = None) -> str | None:
1163
+ """After an agent's touch that began where a notification banner comes down: did another app come to the
1164
+ front? A banner landing between the check before the touch and the touch itself takes it (seen on the phone
1165
+ 2026-11-11: twice Mail opened, once on a toggle at y 159). Then the app the touch was meant for is opened
1166
+ again and the note says so; None when nothing took it. `read`: the app the answer's own read found in
1167
+ front, when it made one (then the phone is asked only if that differs); without one the phone is asked
1168
+ once (about 70 ms, and only after a touch near the top). The home screen or a notification pulled open
1169
+ count too, unless an alert is up (a permission prompt the touch raised is the app's own) or the phone's
1170
+ process list still has the app in front (a banner still sliding over it). `top`: whether the touch began
1171
+ near the top, when it isn't the latest action's (the check at the next call, see _run_tool)."""
1172
+ s = self.session
1173
+ if before is None or not (s.top_touch if top is None else top):
1174
+ return None
1175
+ name, bundle = before
1176
+ if bundle == SPRINGBOARD:
1177
+ # never the app a touch is meant for: a read taken while a banner slides over an app names SpringBoard
1178
+ # (seen on the phone 2026-11-11: "opened" the home screen over Settings), and a touch on the home
1179
+ # screen that opens an app is what it is for
1180
+ return None
1181
+ if read is not None and read[1] == bundle:
1182
+ return None
1183
+ front = await self._read(s.foreground_bundle_id)
1184
+ if not front or front == bundle:
1185
+ return None
1186
+ if await self._still_under(bundle):
1187
+ # the app still runs in front: something lies over it, not instead of it. Sign in with Apple, Apple Pay,
1188
+ # a Safari view, the share sheet are other processes over the app (Tapline's review), and a banner
1189
+ # slides over it. Only SpringBoard staying in front once a banner would have gone is a theft: a shade
1190
+ # (Notification Center, Control Center) pulled down from the top edge
1191
+ if front != SPRINGBOARD:
1192
+ return None
1193
+ if read is None: # the answer had no read of its own: one now, which waits out a banner
1194
+ await self._read(lambda: self._silent_read(view))
1195
+ if view.alert_now is not None or await self._read(s.foreground_bundle_id) != SPRINGBOARD:
1196
+ return None # a permission prompt the touch raised is the app's own; or the banner has gone
1197
+ elif front == SPRINGBOARD:
1198
+ if read is None:
1199
+ await self._read(lambda: self._silent_read(view))
1200
+ if view.alert_now is not None:
1201
+ return None
1202
+ await self.control.agent(lambda: s.launch(bundle))
1203
+ self.log(f"the agent's touch near the top left {bundle} for {front}: opened {bundle} again")
1204
+ if front == SPRINGBOARD:
1205
+ return NOTE_TAKEN_SYSTEM.format(other=SYSTEM_IN_FRONT, app=f"{name} ({bundle})")
1206
+ known = read[0] if read is not None and read[1] == front else s.app_names.get(front)
1207
+ return NOTE_TAKEN.format(other=f"{known or front} ({front})", app=f"{name} ({bundle})")
1208
+
1209
+ async def _still_under(self, bundle: str) -> bool:
1210
+ """Whether iOS's process list still has the app running in front (under whatever WebDriverAgent names)."""
1211
+ try:
1212
+ return bundle in await asyncio.wait_for(self.session.device.foreground_apps(), 5)
1213
+ except Exception:
1214
+ return False # the phone can't say: count the app as gone, the safer side for an agent's next touch
1215
+
1216
+ async def turned(self, seq: int, jpeg: bytes, degrees: int) -> bytes:
1217
+ """A stream frame turned upright for landscape, made once per frame for every viewer."""
1218
+ kept = self._turned
1219
+ if kept is not None and kept[0] == (seq, degrees):
1220
+ return kept[1]
1221
+ out = await asyncio.to_thread(turn_jpeg, jpeg, degrees)
1222
+ self._turned = ((seq, degrees), out)
1223
+ return out
1224
+
1225
+ async def _toggle_landed(self, view, result: "Toggled") -> tuple[Snapshot, float] | None:
1226
+ """A switch's value changes the moment the touch lands; the knob's slide after it is only drawing. Waiting
1227
+ for the screen to stand still cost a toggle 0.33 to 0.65 s (measured on the phone 2026-11-11). So the tree
1228
+ is read until the switch shows its new value, and that read is the answer. Not when anything else
1229
+ appeared or went away in that read (a toggle that opens a section may still be animating it in): then
1230
+ None, and the usual wait for a still screen follows. None too when the switch doesn't flip in
1231
+ TOGGLE_LAND_WAIT."""
1232
+ loop = asyncio.get_running_loop()
1233
+ began = loop.time()
1234
+ old = result.before
1235
+ while loop.time() - began < TOGGLE_LAND_WAIT:
1236
+ snap = await self._read(lambda: self.session.snapshot())
1237
+ if snap.tree.alert is not None:
1238
+ return None # a confirm alert came up: the usual wait, and the verdict knows about alerts
1239
+ now = next((e for e in snap.tree.items if (e.role, e.identifier or "", e.label if not e.identifier else "")
1240
+ == (old.role, old.identifier or "", old.label if not old.identifier else "")), None)
1241
+ if now is not None and now.checked == result.flipped:
1242
+ if len(snap.tree.items) != len(view.items):
1243
+ return None
1244
+ return snap, loop.time() - began
1245
+ await asyncio.sleep(TOGGLE_POLL)
1246
+ return None
1247
+
1248
+ async def _typed_wrong(self, result: "Typed", text: str) -> bool:
1249
+ """Does the field typed into hold something other than `text` on a fresh read? False when it can't be found
1250
+ again (claim nothing) or holds exactly `text`."""
1251
+ snap = await self._read(lambda: self.session.snapshot())
1252
+ old = result.field
1253
+ same = [e for e in snap.tree.items if e.role == old.role and (
1254
+ e.identifier == old.identifier if old.identifier else e.label == old.label)]
1255
+ return len(same) == 1 and not same[0].secure and same[0].value != text
1256
+
1257
+ async def _typed_landed(self, view, result: "Typed", text: str) -> tuple[Snapshot, float] | None:
1258
+ """As _toggle_landed, for typing: the field holds the typed text as soon as the keys call returns, and the
1259
+ screen's wait after it (0.2 to 0.4 s on the phone) only saw the caret blink. The first read that shows the
1260
+ field with exactly that text is the answer, unless anything else appeared or went away in it."""
1261
+ loop = asyncio.get_running_loop()
1262
+ began = loop.time()
1263
+ old = result.field
1264
+ while loop.time() - began < TOGGLE_LAND_WAIT:
1265
+ snap = await self._read(lambda: self.session.snapshot())
1266
+ if snap.tree.alert is not None:
1267
+ return None
1268
+ same = [e for e in snap.tree.items if e.role == old.role and (
1269
+ e.identifier == old.identifier if old.identifier else e.label == old.label)]
1270
+ if len(same) == 1 and not same[0].secure and same[0].value == text:
1271
+ if len(snap.tree.items) != len(view.items):
1272
+ return None
1273
+ return snap, loop.time() - began
1274
+ await asyncio.sleep(TOGGLE_POLL)
1275
+ return None
1276
+
1277
+ async def _apps(self, notes: list) -> dict:
1278
+ apps = await self._read(self.session.apps)
1279
+ if not apps:
1280
+ return _answer(["No apps installed by you."], notes)
1281
+ try:
1282
+ front = await self._read(self.session.foreground_bundle_id)
1283
+ except Refused:
1284
+ raise
1285
+ except Exception:
1286
+ front = None # the list is still right without the marker
1287
+ lines = [app_line(a) + (" *" if a["bundle_id"] == front else "") for a in apps]
1288
+ return _answer([*lines, "* = in front."], notes)
1289
+
1290
+ async def _app(self, query: str, partial: bool = True) -> tuple[str, str]:
1291
+ """(bundle id, how to show it) for what the agent named, in this order: an installed bundle id; a
1292
+ name, ignoring case, among the installed apps (exact, then, if `partial`, the start of a name); a
1293
+ built-in app's name; a bundle-id-looking string as given (built-in apps aren't in `apps`). Parts
1294
+ of bundle ids and of later words never match, so "Notes" doesn't hit "Sticky Notes"."""
1295
+ q = query.strip()
1296
+ apps = await self.session.apps()
1297
+ low = q.lower()
1298
+ for a in apps:
1299
+ if a["bundle_id"] == q:
1300
+ return q, f"{a['name']} ({q})"
1301
+ hits = [a for a in apps if a["name"].lower() == low]
1302
+ builtin = next((n for n in SYSTEM_APPS if n.lower() == low), None)
1303
+ if not hits and builtin:
1304
+ return SYSTEM_APPS[builtin], f"{builtin} ({SYSTEM_APPS[builtin]})"
1305
+ if not hits and partial:
1306
+ hits = [a for a in apps if a["name"].lower().startswith(low)]
1307
+ if not hits:
1308
+ if BUNDLE_ID.match(q):
1309
+ return q, q # launching it says whether it exists
1310
+ raise Refused("not_found", f'no installed app is named "{q}"; call apps to see them')
1311
+ if len(hits) > 1:
1312
+ raise Refused("ambiguous", f'{len(hits)} installed apps match "{q}". Pick one by its bundle id:\n'
1313
+ + "\n".join(app_line(a) for a in hits[:10]))
1314
+ return hits[0]["bundle_id"], f"{hits[0]['name']} ({hits[0]['bundle_id']})"
1315
+
1316
+ async def _open_app(self, view, a: dict) -> str:
1317
+ bundle_id, shown = await self._app(a["app"])
1318
+ try:
1319
+ await self.session.launch(bundle_id)
1320
+ except ConnectionLost:
1321
+ raise
1322
+ except DeviceError as e:
1323
+ raise Refused("not_found", f"couldn't open {bundle_id}: {e}; call apps for the installed apps") from None
1324
+ return f"Opened {shown}."
1325
+
1326
+ async def _close_app(self, view, a: dict) -> str:
1327
+ bundle_id, shown = await self._app(a["app"], partial=False)
1328
+ await self.session.terminate(bundle_id)
1329
+ return f"Closed {shown}."
1330
+
1331
+ async def _close_stuck(self, stuck: AppNotAnswering, query: str, notes: list) -> dict:
1332
+ """close_app while the app in front is frozen: end its processes through iOS (WebDriverAgent's own
1333
+ terminate would wait on the app), then connect again at once. Only that app: the rest can wait."""
1334
+ if self.control.paused:
1335
+ raise Refused("paused", "paused by the user in the live view; ask them to resume")
1336
+ if query.strip().lower() not in (stuck.bundle_id.lower(), stuck.name.lower()):
1337
+ raise self._not_connected()
1338
+ device = self.session.device
1339
+ if device.rsd is None:
1340
+ raise self._not_connected()
1341
+ await device.kill_app(stuck.bundle_id)
1342
+ self.log(f"closed {stuck.bundle_id}, which wasn't answering")
1343
+ self.stuck = None
1344
+ self._wake.set()
1345
+ self._announce("agent", "close_app", label=stuck.name)
1346
+ return _answer([f"Closed {stuck.name} ({stuck.bundle_id}), which wasn't answering."], notes)
1347
+
1348
+ async def _wait(self, view, args: dict, notes: list, shot) -> dict:
1349
+ """Sleep outside the phone's queue (a person may act meanwhile), then answer with what changed."""
1350
+ seconds = args["seconds"]
1351
+ await self.control.sleep(seconds)
1352
+ result = f"Waited {seconds:.1f} s."
1353
+ if not args.get("look", True):
1354
+ view.unlooked += 1
1355
+ return _answer([result], notes)
1356
+ full = view.seen != self.control.generation or view.dropped or view.read_at == float("-inf")
1357
+ answer = await self._after(view, result, notes, full) # no `Settled in`: the app did nothing
1358
+ return await self._add_shot(answer, view, shot)
1359
+
1360
+ async def _wait_for(self, view, args: dict, notes: list, shot) -> dict:
1361
+ """Poll the fast tree (outside the phone's queue between reads) until the text, id or label is
1362
+ on the screen (or, with `gone`, no longer is), then answer with what changed. No `Settled in`."""
1363
+ kind = next(k for k in WAIT_FOR_KINDS if args.get(k) is not None)
1364
+ target = ui.parse_target({kind: args[kind]})
1365
+ gone, timeout = bool(args.get("gone", False)), float(args.get("timeout", WAIT_FOR_DEFAULT))
1366
+ clock = self.control.clock
1367
+ began = clock()
1368
+ timed_out = Refused("timeout", f'Waited {timeout:g} s for {kind} "{args[kind]}" to '
1369
+ f"{'disappear' if gone else 'appear'}; it didn't happen.")
1370
+ seen = False # has the screen been read at all during this wait?
1371
+ while True:
1372
+ left = timeout - (clock() - began)
1373
+ if left < WAIT_FOR_MIN_LEFT:
1374
+ raise timed_out
1375
+ # The read gets its own full deadline, so a slow phone isn't taken for a dead one; wait_for's
1376
+ # deadline is kept by cancelling the read, which answers `timeout` and leaves the link alone.
1377
+ # "It didn't happen" needs one look at the screen, so the first read gets at least
1378
+ # WAIT_FOR_FIRST_READ seconds, whatever the wait's own time: over Wi-Fi one read can take most of a
1379
+ # second, and a 0.5 s wait then never saw the screen at all (seen 2026-11-10: twelve of twelve). If
1380
+ # even that read doesn't come back (a stall), the answer says that the screen wasn't read, not what
1381
+ # was on it (the same day: a 5 s wait said a text didn't appear which had been there all along).
1382
+ grace = left if seen else max(left, WAIT_FOR_FIRST_READ)
1383
+ try:
1384
+ tree = await asyncio.wait_for(
1385
+ self._read(lambda: self.session.refresh(timeout=WAIT_FOR_READ_MAX)), grace)
1386
+ except asyncio.TimeoutError:
1387
+ if not seen:
1388
+ raise Refused("timeout", f"The phone didn't answer within {max(timeout, WAIT_FOR_FIRST_READ):g} s, so it "
1389
+ f'isn\'t known whether {kind} "{args[kind]}" is on screen; look, then '
1390
+ "wait again.") from None
1391
+ raise timed_out from None
1392
+ seen = True
1393
+ if self._missing(tree.items, target) == gone:
1394
+ break
1395
+ left = timeout - (clock() - began)
1396
+ if left < WAIT_FOR_MIN_LEFT:
1397
+ raise timed_out
1398
+ await self.control.sleep(min(WAIT_FOR_POLL, left))
1399
+ result = f"Done after {clock() - began:.1f} s."
1400
+ if not args.get("look", True):
1401
+ view.unlooked += 1
1402
+ return _answer([result], notes)
1403
+ full = view.seen != self.control.generation or view.dropped or view.read_at == float("-inf")
1404
+ # the last poll is the answer's tree: no read of its own, only the app lookup the polls left out
1405
+ snap = await self._read(lambda: self.session.snapshot_from(tree))
1406
+ return await self._add_shot(await self._after(view, result, notes, full, snap=snap), view, shot)
1407
+
1408
+ async def _type_text(self, view, a: dict) -> Typed:
1409
+ """Type into the target, or into the only input on screen (the fast tree has no focus attribute).
1410
+ The target is tapped first by the session, which waits for the keyboard and for focus to arrive."""
1411
+ target = a.get("target")
1412
+ if target is None:
1413
+ await self._silent_read(view) # which inputs there are must be read now
1414
+ inputs = [e for e in view.items if e.role == "input"]
1415
+ if not inputs:
1416
+ raise Refused("not_found", "No input on screen")
1417
+ if len(inputs) > 1:
1418
+ shown = "\n".join([*(f" {e.line()}" for e in inputs[:ui.MAX_CANDIDATES]),
1419
+ *ui.more_candidates(len(inputs), ui.MAX_CANDIDATES),
1420
+ "Retry with one of the refs as the target."]) # no target, so no `index` to add
1421
+ raise Refused("ambiguous", f"{len(inputs)} inputs on screen; pass a target to pick one:\n"
1422
+ f"Candidates:\n{shown}")
1423
+ field = inputs[0]
1424
+ elif target.point is not None:
1425
+ x, y = target.point
1426
+ field = ui.at_point(view.items, target.point, "input")
1427
+ if field is None:
1428
+ raise Refused("not_actionable", f"no_action: Nothing at ({round(x)}, {round(y)}) takes typing.")
1429
+ else:
1430
+ field = ui.resolve(view.items, target)
1431
+ if field.role != "input":
1432
+ raise Refused("not_actionable", f"no_action: {field.line(actions=False)} is not an input")
1433
+ # the focusing tap is a touch: refused where tap would be, and made where it shows; `force` as for tap
1434
+ self._refuse_disabled(field)
1435
+ forced, at = "", None
1436
+ if a.get("force"):
1437
+ forced = self._forced(view, field)
1438
+ else:
1439
+ self._guard(view, field)
1440
+ at = self._touch_point(field)
1441
+ async def typed() -> None:
1442
+ await self.session.type_text(a["text"], replace=bool(a.get("replace", True)),
1443
+ submit=bool(a.get("submit", False)), target=ui.Target(ref=field.ref), at=at)
1444
+ try:
1445
+ await typed()
1446
+ except Refused as e:
1447
+ # The keyboard was left up for the tap on another input, and this app gave that tap to the keyboard
1448
+ # (it closed it, or nothing happened): close it the old way and tap again. Nothing was typed.
1449
+ if not (view.read_keyboard and "didn't take focus" in str(e)):
1450
+ raise
1451
+ await self._close_keyboard(view, "type_text", ui.Target(ref=field.ref), shown_inputs=True)
1452
+ await typed()
1453
+ done = Typed(f"Typed into {field.line(actions=False)}.")
1454
+ retyped = NOTE_RETYPED if getattr(self.session, "retyped", False) else ""
1455
+ done.field, done.note = field, " ".join(n for n in (forced, retyped) if n)
1456
+ return done
1457
+
1458
+ async def _scroll(self, view, a: dict) -> Scrolled:
1459
+ """Swipe through the centre of the scroll area (the innermost one around the target, the point or the
1460
+ screen's centre), `amount` of its height (or width), between the bars. `to` repeats that until a swipe
1461
+ leaves the settled screen as the one before left it (a rubber band or a spinner moves the screen
1462
+ meanwhile, so "no change at all" would never come), at most MAX_SCROLL_SWIPES times or SCROLL_CAP seconds."""
1463
+ s = self.session
1464
+ target, to = a.get("target"), a.get("to")
1465
+ if to == "target":
1466
+ return await self._scroll_to(view, target)
1467
+ if target is None:
1468
+ area = ui.scroll_container(view.items, s.screen)
1469
+ elif target.point is not None:
1470
+ x, y = target.point
1471
+ area = ui.scroll_at(view.items, target.point)
1472
+ if area is None:
1473
+ raise Refused("not_actionable", f"no_action: Nothing at ({round(x)}, {round(y)}) takes a scroll.")
1474
+ else:
1475
+ hit = ui.resolve(view.items, target)
1476
+ area = ui.scroll_of(view.items, hit)
1477
+ if area is None:
1478
+ raise Refused("not_actionable", f"no_action: {hit.line(actions=False)} is not inside a scroll view")
1479
+ direction = a.get("direction") or (SCROLL_TO[to][0] if to else "down") # _checked made `to` and it agree
1480
+ top, bottom = s.last_insets
1481
+ band = (SCROLL_TOP_INSET if top is None else top,
1482
+ s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom))
1483
+ x0, y0, x1, y1 = ui.scroll_area_points(direction, area.rect if area else (0.0, 0.0, *s.screen),
1484
+ a.get("amount", 0.8), band)
1485
+ bounds = area.rect if area else (0.0, 0.0, *s.screen)
1486
+ # Whether the tree shows what lies past the list's edge. Where it does (React Native renders a whole list),
1487
+ # the end is reached when a read after a drag shows nothing past it any more: no waiting for the pictures
1488
+ # to repeat, which over Wi-Fi took 4 to 14 s. A list that keeps only its visible rows shows nothing past
1489
+ # its edge either way: it is scrolled as before, until the pictures stop changing.
1490
+ beyond = bool(to) and direction in ("up", "down") and self._more_beyond(direction, bounds)
1491
+ other = {"up": "down", "down": "up"}.get(direction)
1492
+ if to and other and not beyond and self._more_beyond(other, bounds):
1493
+ # The tree shows this list's rows past its other edge, so it would show them past this one too: there
1494
+ # are none, so the list is already at its start (or end). No drag, and no wait for pictures to repeat
1495
+ # (13 s over Wi-Fi, 2026-11-11). (A list that keeps only its visible rows shows none either way and
1496
+ # is scrolled as before.)
1497
+ return Scrolled(f"{area.line(actions=False) if area else 'The screen'} is already at its {to}.")
1498
+ loop = asyncio.get_running_loop()
1499
+ began, previous, note = loop.time(), self._mark()[1], ""
1500
+ for swipes in range(1, MAX_SCROLL_SWIPES + 1):
1501
+ seq, since = self._mark()
1502
+ await s.scroll(x0, y0, x1, y1)
1503
+ if not to or swipes == MAX_SCROLL_SWIPES:
1504
+ break
1505
+ await self._settle(seq, cap=RESOLVE_SETTLE, since=since)
1506
+ settled = self._mark()[1]
1507
+ if beyond:
1508
+ await self._silent_read(view)
1509
+ if not self._more_beyond(direction, bounds):
1510
+ break # the tree said there was more past that edge, and now there is none: reached
1511
+ if alike(settled, previous):
1512
+ break # the swipe left the screen as the one before did: the end of the list
1513
+ previous = settled # (with no stream there is nothing to compare: every swipe is made)
1514
+ if loop.time() - began >= SCROLL_CAP:
1515
+ # the screen still changed after every swipe: more may be loading, or the list is long
1516
+ note = f"the {to} wasn't reached in {SCROLL_CAP:g} s; scroll again to go on."
1517
+ break
1518
+ done = Scrolled(f"Scrolled {area.line(actions=False) if area else 'the screen'} {direction}.")
1519
+ done.note = note
1520
+ return done
1521
+
1522
+ def _more_beyond(self, direction: str, bounds: tuple[float, float, float, float]) -> bool:
1523
+ """Does the last read have elements past the scroll area's top (`up`) or bottom (`down`) edge, within its
1524
+ width? What a list holds beyond its edge is in the tree (React Native renders it, UIKit keeps nearby rows);
1525
+ an empty answer means its start (or end) is on screen."""
1526
+ x, y, w, h = bounds
1527
+ for e in getattr(self.session, "offscreen", []):
1528
+ if e.x + e.width <= x or e.x >= x + w:
1529
+ continue # beside the area (a carousel's cards), not past its edge
1530
+ # by its middle: a row that straddles the edge with its middle past it is still past it (seen over
1531
+ # Wi-Fi 2026-11-11: Bottom at y 915 to 963 on a 932 pt screen, and the scroll stopped one drag short)
1532
+ if direction == "up" and e.center[1] < y:
1533
+ return True
1534
+ if direction == "down" and e.center[1] >= y + h:
1535
+ return True
1536
+ return False
1537
+
1538
+ def _anywhere(self, view, target: ui.Target) -> ui.Element:
1539
+ """The element a target names, on the agent's list or, failing that, among what the last read found off
1540
+ screen. Raises as `resolve` does when it is in neither."""
1541
+ off = getattr(self.session, "offscreen", [])
1542
+ if target.text is not None:
1543
+ # {text} is an exact match first, then a part of one: an element off screen that is exactly the text
1544
+ # comes before one on screen that only contains it (seen on the phone 2026-11-10: "Alert" found the
1545
+ # state line, which reads "… alert=none …", and the Alert button below the screen was never reached)
1546
+ want = ui._norm(target.text)
1547
+ if (not any(want in (ui._norm(e.label), ui._norm(e.value)) for e in view.items)
1548
+ and any(want in (ui._norm(e.label), ui._norm(e.value)) for e in off)):
1549
+ return ui.resolve(sorted([*view.items, *off], key=lambda e: (e.y, e.x)), target)
1550
+ try:
1551
+ return ui.resolve(view.items, target)
1552
+ except Refused as e:
1553
+ if e.code != "not_found" or target.ref is not None:
1554
+ raise
1555
+ # on screen and off it together, in reading order: an `index` counts through both, and several
1556
+ # matches without one are `ambiguous` (the first off-screen one may lie the other way)
1557
+ everything = sorted([*view.items, *off], key=lambda e: (e.y, e.x))
1558
+
1559
+ return ui.resolve(everything, target)
1560
+
1561
+ async def _scroll_to(self, view, target: ui.Target) -> Scrolled:
1562
+ """`scroll` with `to: "target"` (AGENT-TOOLS v1.6): drag the scroll view that holds the element until the
1563
+ element is on screen and clear of the bars. The tree gives the place of an element that is off screen, and
1564
+ an eased drag doesn't slide on, so each drag covers a known distance; the screen is read after each one,
1565
+ and the next drag covers what is left. It stops when the element is clear, when a drag no longer moves it
1566
+ (the end of the list), or at the limits `to: "end"` has. A row a lazy list hasn't made yet is `not_found`."""
1567
+ s = self.session
1568
+ loop = asyncio.get_running_loop()
1569
+ began, note, where, clear = loop.time(), "", None, False
1570
+ seen: list[tuple[float, ...]] = [] # where the element was before each drag
1571
+ for drags in range(MAX_SCROLL_SWIPES + 1):
1572
+ try:
1573
+ element = self._anywhere(view, target)
1574
+ except Refused:
1575
+ if seen: # the screen has moved under the agent's list, and this answer carries none
1576
+ view.moved = True
1577
+ raise
1578
+ area = ui.scroll_holding(view.items, element, s.screen)
1579
+ where = where or (area.line(actions=False) if area else "the screen")
1580
+ top, bottom = s.last_insets
1581
+ band = (SCROLL_TOP_INSET if top is None else top,
1582
+ s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom))
1583
+ x, y, w, h = area.rect if area else (0.0, 0.0, *s.screen)
1584
+ y, h = max(y, band[0]), min(y + h, band[1]) - max(y, band[0])
1585
+ if h <= 0: # the scroll view itself lies under the bars: nothing to drag in
1586
+ note = "it doesn't scroll further: the target is as far into view as it gets."
1587
+ break
1588
+ move = ui.scroll_move(element.rect, (x, y, w, h))
1589
+ # after a drag, inside the view is enough: chasing the last points of margin with a drag that can't
1590
+ # be shorter than SCROLL_MIN_DRAG would go back and forth
1591
+ if move == (0.0, 0.0) or (seen and ui.scroll_move(element.rect, (x, y, w, h), margin=0.0) == (0.0, 0.0)):
1592
+ clear = True
1593
+ break
1594
+ if any(ui._same_rect(was, element.rect) for was in seen):
1595
+ note = "it doesn't scroll further: the target is as far into view as it gets."
1596
+ break
1597
+ if loop.time() - began >= SCROLL_CAP:
1598
+ note = f"the target wasn't reached in {SCROLL_CAP:g} s; scroll again to go on."
1599
+ break
1600
+ if drags == MAX_SCROLL_SWIPES:
1601
+ note = f"the target wasn't reached in {MAX_SCROLL_SWIPES} drags; scroll again to go on."
1602
+ break
1603
+ seen.append(element.rect)
1604
+ await s.scroll(*ui.scroll_step(move, (x, y, w, h)))
1605
+ await self._silent_read(view) # once the screen is still: where the element is now
1606
+ line = element.line(ref=bool(element.ref), actions=False)
1607
+ if clear and not seen: # nothing was dragged: "Scrolled" would read as if the layout had moved
1608
+ return Scrolled(f"{line} is already on screen.")
1609
+ done = Scrolled(f"Scrolled {where} to {line}." if clear else f"Scrolled {where} toward {line}.")
1610
+ done.note = note
1611
+ return done
1612
+
1613
+ async def _outside_focus(self, view, name: str, target: ui.Target, keyboard_top: float,
1614
+ shown_inputs: bool = False) -> bool:
1615
+ """Is the touch's target something other than the focused input, so the keyboard is in the way of it
1616
+ (AGENT-TOOLS v1.4)? False for the focused input itself, an element in an OS alert, a point on the
1617
+ keyboard (a key), and whatever the tool is about to refuse (nothing is touched then). False too for
1618
+ `type_text` into another input that shows above the keyboard: a tap there moves the focus, as a person's
1619
+ does when filling a form, and closing the keyboard between two fields cost two seconds a field (measured
1620
+ on the phone 2026-11-10: 3.4 s against 1.7 s). `shown_inputs` asks about those as well."""
1621
+ items = view.items
1622
+ if target.point is not None:
1623
+ x, y = target.point
1624
+ if name != "type_text" and y >= keyboard_top:
1625
+ return False
1626
+ if any(e.in_alert and e.x <= x < e.x + e.width and e.y <= y < e.y + e.height for e in items):
1627
+ return False
1628
+ field = ui.at_point(items, target.point, "input")
1629
+ else:
1630
+ try:
1631
+ field = ui.resolve(items, target)
1632
+ except Refused:
1633
+ return False
1634
+ if isinstance(field, tuple) or field.in_alert:
1635
+ return False
1636
+ if name == "type_text" and (field is None or field.role != "input"):
1637
+ return False # it refuses typing into this itself
1638
+ if name == "type_text" and not shown_inputs and keyboard_top - field.y >= MIN_VISIBLE:
1639
+ # at least 10 pt of it show above the keyboard: a person taps that part and focus moves (AGENT-TOOLS
1640
+ # §4, v1.6; the lab's second field shows 17 pt, and closing first cost 2 s there)
1641
+ return False
1642
+ return field is None or not await self.session.is_focused(field) # a non-input is outside by definition
1643
+
1644
+ async def _close_keyboard(self, view, name: str, target: ui.Target | None, shown_inputs: bool = False) -> str:
1645
+ """Before a touch (or a focusing tap) on something other than the focused input while the keyboard is up:
1646
+ close it the way a person does, since the first tap outside the field only closes it and the keyboard
1647
+ covers what is near the bottom. One touch where nothing responds, then WebDriverAgent's own dismissal
1648
+ if it is still up; neither presses return. Gives the note when it ended up closed, else leaves the
1649
+ screen to the usual `covered` rules."""
1650
+ s = self.session
1651
+ if name not in KEYBOARD_TOOLS or target is None or not view.read_keyboard or view.alert_now is not None:
1652
+ return "" # (an OS alert in front is modal: a touch outside it closes nothing, and behind_alert decides)
1653
+ top, bottom = s.last_insets
1654
+ top = SCROLL_TOP_INSET if top is None else top
1655
+ bottom = s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom)
1656
+ if not await self._outside_focus(view, name, target, bottom, shown_inputs):
1657
+ return ""
1658
+ return await self._dismiss_keyboard_now(view)
1659
+
1660
+ async def _dismiss_keyboard_now(self, view) -> str:
1661
+ """Close the keyboard the way a person does: one touch where nothing responds, then WebDriverAgent's own
1662
+ dismissal if it is still up; neither presses return. NOTE_KEYBOARD when it ended up closed, else ""."""
1663
+ s = self.session
1664
+ top, bottom = s.last_insets
1665
+ top = SCROLL_TOP_INSET if top is None else top
1666
+ bottom = s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom)
1667
+ if (blank := ui.blank_point(view.items, s.screen, top, bottom)) is not None:
1668
+ await s.tap(ui.Target(x=blank[0], y=blank[1]))
1669
+ await self._silent_read(view)
1670
+ if view.read_keyboard and await s.dismiss_keyboard():
1671
+ await self._silent_read(view)
1672
+ return "" if view.read_keyboard else NOTE_KEYBOARD
1673
+
1674
+ def _under_keyboard(self, target: ui.Target) -> bool:
1675
+ """Is the target an element the last read found past its scroll view's edge but still on the phone's
1676
+ screen: hidden by the keyboard rather than scrolled away?"""
1677
+ try:
1678
+ hit = ui.resolve(getattr(self.session, "offscreen", []), dataclasses.replace(target, index=0))
1679
+ except Refused:
1680
+ return False
1681
+ return isinstance(hit, ui.Element) and 0 <= hit.y < self.session.screen[1] and hit.y + hit.height > 0
1682
+
1683
+ @staticmethod
1684
+ def _refuse_behind_alert(view, force: bool = True) -> None:
1685
+ """When the newest read shows an OS alert: `behind_alert`. `force` says whether passing it could help
1686
+ (it can't touch an element that isn't found)."""
1687
+ if view.alert_now is not None:
1688
+ title, buttons = view.alert_now
1689
+ alert = " ".join([json.dumps(title, ensure_ascii=False), *(f"[{b}]" for b in buttons)])
1690
+ raise Refused("not_actionable", f"behind_alert: an alert covers it: {alert}; answer it with alert, "
1691
+ + ("or pass force: true." if force else "or ask the user."))
1692
+
1693
+ def _refuse_behind_sheet(self, target: ui.Target) -> None:
1694
+ """A target on no listed element but on one an open sheet (or another layer) hides: `covered`, naming what
1695
+ is in front, not `not_found`. It isn't on the list, so `force` has nothing to touch."""
1696
+ behind = getattr(self.session, "behind", [])
1697
+ try:
1698
+ hit = ui.resolve([e for e, _ in behind], dataclasses.replace(target, index=0))
1699
+ except Refused:
1700
+ return
1701
+ for e, front in behind:
1702
+ if e is hit:
1703
+ raise Refused("not_actionable", f"covered: {hit.line(ref=False, actions=False)} is behind "
1704
+ f"{self._front_line(front)}; close that first.")
1705
+
1706
+ def _front_line(self, front: ui.Element | None) -> str:
1707
+ """The line of what is open in front, with the ref the agent's list has for it (the read that found it
1708
+ numbered it for itself); with no ref when the agent's list doesn't have it."""
1709
+ if front is None:
1710
+ return "what is open in front of it"
1711
+ plain = front.line(ref=False, actions=False)
1712
+ known = next((e for e in self.session.items if e.line(ref=False, actions=False) == plain), None)
1713
+ return known.line(actions=False) if known is not None else plain
1714
+
1715
+ def _still_open(self) -> str:
1716
+ """For a `back` that changed nothing: what is open on top, if the last read found a layer in front."""
1717
+ behind = getattr(self.session, "behind", [])
1718
+ if not behind:
1719
+ return "."
1720
+ front = behind[0][1]
1721
+ named = self._front_line(front) if front is not None else "a sheet or menu"
1722
+ return f"; {named} is still open: tap its close control."
1723
+
1724
+ def _refuse_off_screen(self, target: ui.Target) -> None:
1725
+ """A target on no listed element but on one the last read found past the screen (or its scroll view's
1726
+ edge): `off_screen`, not `not_found`. A touch can't reach it, so `force` doesn't either."""
1727
+ try:
1728
+ hit = ui.resolve(getattr(self.session, "offscreen", []), dataclasses.replace(target, index=0))
1729
+ except Refused:
1730
+ return
1731
+ if isinstance(hit, ui.Element):
1732
+ raise Refused("not_actionable", f"off_screen: {hit.line(ref=False, actions=False)} is off screen; "
1733
+ f"{SCROLL_HINT}")
1734
+
1735
+ @staticmethod
1736
+ def _refuse_disabled(element: ui.Element) -> None:
1737
+ if not element.enabled:
1738
+ raise Refused("not_actionable", f"disabled: {element.line(actions=False)} is disabled.")
1739
+
1740
+ def _guard(self, view, element: ui.Element) -> None:
1741
+ """A9: refuse what a real touch couldn't do. The reason word comes first in the message."""
1742
+ self._refuse_disabled(element)
1743
+ if not element.in_alert:
1744
+ self._refuse_behind_alert(view)
1745
+
1746
+ def _touch_point(self, element: ui.Element) -> tuple[float, float] | None:
1747
+ """Where a finger would touch an element a bar (or the keyboard) is drawn over its centre: the middle of
1748
+ the part that shows after every overlapping bar is cut away, nearest the centre. None when no bar
1749
+ covers the centre. Refuses when too little shows."""
1750
+ centre = element.center
1751
+ bars = element.covered_by
1752
+ width, height = self.session.screen or (0.0, 0.0)
1753
+ if height > width and element.rect[1] < STATUS_BAR:
1754
+ bars = (*bars, ui.Bar("StatusBar", (0.0, 0.0, width, STATUS_BAR), "the status bar"))
1755
+ over = [b for b in bars if b.rect[0] <= centre[0] < b.rect[0] + b.rect[2]
1756
+ and b.rect[1] <= centre[1] < b.rect[1] + b.rect[3]]
1757
+ if not over:
1758
+ return None
1759
+ x, y, w, h = ui.visible_part(element.rect, bars, self.session.screen)
1760
+ if w >= MIN_VISIBLE and h >= MIN_VISIBLE:
1761
+ return x + w / 2, y + h / 2
1762
+ line = element.line(actions=False)
1763
+ if over[0].kind == "StatusBar":
1764
+ raise Refused("not_actionable", f"covered: {line} is under the status bar, where a touch scrolls the "
1765
+ f"list to its top; {SCROLL_HINT}")
1766
+ if over[0].kind == "TabBar":
1767
+ raise Refused("not_actionable", f"under_tab_bar: {line} is under the tab bar iOS draws; {SCROLL_HINT}")
1768
+ if over[0].kind in ui.COVERING_BARS:
1769
+ raise Refused("not_actionable", f"covered: {line} is covered by {over[0].line}; {SCROLL_HINT}")
1770
+ raise Refused("not_actionable", f"covered: {line} is covered by {over[0].line}; move or close it first.")
1771
+
1772
+ async def _touch(self, view, a: dict, hold_ms: int) -> str:
1773
+ """`tap` and `long_press`: refuse a disabled element or one behind an alert, and touch the part of an
1774
+ element under a bar that shows. `force` skips all of it. A point is a raw touch."""
1775
+ s, target = self.session, _target(a)
1776
+ verb = "Long-pressed" if hold_ms > 300 else "Tapped"
1777
+ found = ui.resolve(s.items, target)
1778
+ if isinstance(found, tuple):
1779
+ return await s.tap(target, hold_ms=hold_ms)
1780
+ self._refuse_disabled(found) # even `force` doesn't touch what the app has switched off
1781
+ if a.get("force"):
1782
+ done = Noted(await s.tap(target, hold_ms=hold_ms)) # at its centre, whatever is drawn there
1783
+ done.note = self._forced(view, found)
1784
+ return done
1785
+ self._guard(view, found)
1786
+ if (point := self._touch_point(found)) is None:
1787
+ return await s.tap(target, hold_ms=hold_ms)
1788
+ await s.tap(ui.Target(x=point[0], y=point[1]), hold_ms=hold_ms)
1789
+ return f"{verb} {found.line(actions=False)} at its visible part ({round(point[0])}, {round(point[1])})."
1790
+
1791
+ def _forced(self, view, element: ui.Element) -> str:
1792
+ """The note for a touch made with `force` at the centre of an element that the checks would have refused,
1793
+ or whose centre a bar is drawn over (there the unforced touch goes to the part that shows): with a real
1794
+ touch, `force` doesn't make the hidden element act. Empty when nothing was in the way."""
1795
+ reason = ""
1796
+ try:
1797
+ if not element.in_alert:
1798
+ self._refuse_behind_alert(view)
1799
+ except Refused as e:
1800
+ reason = str(e).split(":", 1)[0]
1801
+ try:
1802
+ if not reason and self._touch_point(element) is not None:
1803
+ centre = element.center
1804
+ over = [b for b in element.covered_by if b.rect[0] <= centre[0] < b.rect[0] + b.rect[2]
1805
+ and b.rect[1] <= centre[1] < b.rect[1] + b.rect[3]]
1806
+ reason = "under_tab_bar" if over and over[0].kind == "TabBar" else "covered"
1807
+ except Refused as e:
1808
+ reason = str(e).split(":", 1)[0]
1809
+ return NOTE_FORCED.format(reason=reason) if reason else ""
1810
+
1811
+ async def _tap(self, view, a: dict) -> str:
1812
+ return await self._touch(view, a, TAP_MS)
1813
+
1814
+ async def _long_press(self, view, a: dict) -> str:
1815
+ return await self._touch(view, a, int(float(a.get("seconds", 1.0)) * 1000))
1816
+
1817
+ async def _toggle(self, view, a: dict) -> str:
1818
+ target = _target(a)
1819
+ if target.point is not None:
1820
+ x, y = target.point
1821
+ found = ui.at_point(view.items, target.point, "switch")
1822
+ if found is None:
1823
+ raise Refused("not_actionable", f"no_action: Nothing at ({round(x)}, {round(y)}) takes a toggle.")
1824
+ else:
1825
+ found = self._switch_first(view.items, target)
1826
+ line = found.line(actions=False)
1827
+ if found.role != "switch" or found.checked is None:
1828
+ raise Refused("not_actionable", f"no_action: {line} is not a switch")
1829
+ self._refuse_disabled(found)
1830
+ forced, point, shown = "", None, None
1831
+ if a.get("force"):
1832
+ forced = self._forced(view, found)
1833
+ else:
1834
+ self._guard(view, found)
1835
+ point = shown = self._touch_point(found)
1836
+ if found.width > WIDE_SWITCH * found.height:
1837
+ point = self._switch_control(view.items, found, shown)
1838
+ want = a.get("value")
1839
+ if want is not None and found.checked == want:
1840
+ return Unchanged(f"{line} is already {'true' if want else 'false'}.")
1841
+ await self.session.tap(ui.Target(ref=found.ref) if point is None else ui.Target(x=point[0], y=point[1]))
1842
+ # Until the tree is read, claim no new state: the first line is the verdict's to write
1843
+ where = "" if shown is None else f" at its visible part ({round(point[0])}, {round(point[1])})"
1844
+ done = Toggled(f"Tapped {dataclasses.replace(found, checked=None, value='').line(actions=False)}{where}.")
1845
+ done.ref, done.before, done.want, done.flipped = found.ref, found, want, not found.checked
1846
+ done.note = forced
1847
+ return done
1848
+
1849
+ def _switch_control(self, items: list[ui.Element], switch: ui.Element,
1850
+ shown: tuple[float, float] | None) -> tuple[float, float] | None:
1851
+ """Where the control of a switch that is a whole row sits (iOS Settings: "Allow Notifications" is one
1852
+ element 350x29 at x 40; a tap on its middle lands on the label and toggles nothing, seen on the phone
1853
+ 2026-11-11). The control is at the end away from the label: the right in a left-to-right layout, the
1854
+ left in Arabic or Hebrew. The row's label shows as its own element inside it; without one, the right.
1855
+ Partly covered (`shown`, the visible part's middle when the centre is covered): the control's end at that
1856
+ height. A bar over the control itself is refused `covered`, as tap refuses: a finger couldn't reach it."""
1857
+ x, y, w, h = switch.rect
1858
+ label = next((e for e in items if e is not switch and e.label and e.label == switch.label
1859
+ and x <= e.center[0] <= x + w and y <= e.center[1] <= y + h), None)
1860
+ leading_right = label is not None and label.center[0] > x + w / 2
1861
+ point = (x + SWITCH_FROM_RIGHT if leading_right else x + w - SWITCH_FROM_RIGHT,
1862
+ y + h / 2 if shown is None else shown[1])
1863
+ over = next((b for b in switch.covered_by if ui._holds(b.rect, point)), None)
1864
+ if over is not None:
1865
+ hint = SCROLL_HINT if over.kind in ui.COVERING_BARS else "move or close it first."
1866
+ raise Refused("not_actionable", f"covered: {switch.line(actions=False)} is covered by {over.line}; {hint}")
1867
+ return point
1868
+
1869
+ @staticmethod
1870
+ def _switch_first(items: list[ui.Element], target: ui.Target) -> ui.Element:
1871
+ """The element a target names, preferring a switch: a row and its switch often share a label."""
1872
+ if target.ref is None:
1873
+ try:
1874
+ return ui.resolve([e for e in items if e.role == "switch"], target)
1875
+ except Refused as e:
1876
+ if e.code != "not_found":
1877
+ raise # several switches match: the agent picks one
1878
+ return ui.resolve(items, target)
1879
+
1880
+ async def _back(self, view, a: dict) -> str:
1881
+ """Close an open popover by a touch outside it (AGENT-TOOLS v1.4); else tap the navigation bar's back
1882
+ button. A bar without one is a root screen (or has only Cancel,
1883
+ Done and the like): nothing is tapped. The edge swipe is only for screens with no bar at all."""
1884
+ s = self.session
1885
+ back = await self._still_back()
1886
+ if back.dismiss is not None: # AGENT-TOOLS v1.4: what is open on top closes first, as "up one screen"
1887
+ await s.tap(ui.Target(x=back.dismiss[0], y=back.dismiss[1]))
1888
+ return "Closed the sheet."
1889
+ behind = getattr(s, "behind", [])
1890
+ # A sheet in front with no region of its own to tap. A back button it hides belongs to the screen behind
1891
+ # (it can't be reached, and where the backdrop lets a touch through it would leave that screen with the
1892
+ # sheet open); one that shows is the sheet's own and is tapped as usual.
1893
+ if behind and not (back.point is not None and not any(ui._holds(e.rect, back.point) for e, _ in behind)):
1894
+ return await self._close_sheet(view)
1895
+ if back.point is not None:
1896
+ await s.tap(ui.Target(x=back.point[0], y=back.point[1]))
1897
+ return "Tapped the back button."
1898
+ if back.bar:
1899
+ raise Refused("not_actionable", "no_action: the navigation bar has no back button")
1900
+ width, height = s.screen
1901
+ await s.swipe(2, height / 2, 0.6 * width, height / 2, 300)
1902
+ return "Swiped from the left edge."
1903
+
1904
+ async def _still_back(self) -> ui.Back:
1905
+ """Where `back` would touch, read until two reads in a row agree. A sheet that is still sliding in has
1906
+ more room above it than it will have: the middle of that room, tapped half a second later, is inside the
1907
+ sheet. On the phone that touch landed on an app in the share sheet and opened it (2026-11-10, over Wi-Fi,
1908
+ where a read and the touch after it are further apart; Mail came to the front). So nothing is touched
1909
+ until the sheet stands still; a sheet that never does is `no_action`."""
1910
+ s = self.session
1911
+
1912
+ def where(back: ui.Back) -> tuple:
1913
+ behind = getattr(s, "behind", [])
1914
+ front = behind[0][1] if behind else None
1915
+ box = ui.sheet_box(front, getattr(s, "front_items", None) or [], s.screen) if behind else None
1916
+ return back.dismiss, back.point, box
1917
+ back = await s.back_point()
1918
+ seen = where(back)
1919
+ if back.dismiss is None and not getattr(s, "behind", []):
1920
+ return back # nothing is open on top: a bar's back button doesn't slide
1921
+ for _ in range(BACK_STILL_TRIES):
1922
+ await asyncio.sleep(BACK_STILL_PAUSE)
1923
+ back = await s.back_point()
1924
+ now = where(back)
1925
+ if now == seen:
1926
+ return back
1927
+ seen = now
1928
+ raise Refused("not_actionable", "no_action: what is open on top is still moving; call back again in a moment")
1929
+
1930
+ async def _close_sheet(self, view) -> str:
1931
+ """Close the sheet in front the two ways a person tries from outside it: a touch beside it, on what it
1932
+ dims, then pulling it down from its top edge. After each, the screen is read: the sheet is closed when the
1933
+ layer in front is gone or is another one. A sheet that takes neither is `no_action`, naming it."""
1934
+ s = self.session
1935
+
1936
+ def front() -> str | None:
1937
+ behind = getattr(s, "behind", [])
1938
+ if not behind:
1939
+ return None
1940
+ return behind[0][1].line(ref=False, actions=False) if behind[0][1] is not None else ""
1941
+ was = front()
1942
+ box = ui.sheet_box(getattr(s, "behind")[0][1], getattr(s, "front_items", None) or view.items, s.screen)
1943
+ if box is not None:
1944
+ top, bottom = s.last_insets
1945
+ beside = ui.beside_sheet(box, s.screen, SCROLL_TOP_INSET if top is None else top,
1946
+ s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom))
1947
+ if beside is not None:
1948
+ await s.tap(ui.Target(x=beside[0], y=beside[1]))
1949
+ await self._silent_read(view)
1950
+ if front() != was:
1951
+ return "Closed the sheet."
1952
+ if (pull := ui.sheet_pull(box, s.screen)) is not None:
1953
+ await s.swipe(*pull, 300)
1954
+ await self._silent_read(view)
1955
+ if front() != was:
1956
+ return "Closed the sheet."
1957
+ raise Refused("not_actionable", "no_action: nothing changed after back" + self._still_open())
1958
+
1959
+ async def _settle_pending(self) -> None:
1960
+ """XCTest no longer waits for the app to go idle, so after an action that wasn't settled (`look: false`)
1961
+ nothing may act or read until the screen is still (cap RESOLVE_SETTLE)."""
1962
+ if self.unsettled:
1963
+ await self._settle(self._mark()[0], cap=RESOLVE_SETTLE)
1964
+ self.unsettled = False
1965
+
1966
+ async def _silent_read(self, view) -> bool:
1967
+ """Read the live tree for target resolution, once the screen is still (cap RESOLVE_SETTLE; with no
1968
+ frames that is a `settle_fallback` wait, 0.6 s). Read mid-redraw, an element can be briefly missing.
1969
+ True when a re-read could show something else: the stream moved, ended capped, or has no frames."""
1970
+ s = self.session
1971
+ done = await self._settle(self._mark()[0], cap=RESOLVE_SETTLE)
1972
+ self.unsettled = False
1973
+ mark = self._mark() # taken before the read: a frame that changes meanwhile counts against the list
1974
+ tree = await s.refresh()
1975
+ view.alert_now = _alert_key(tree.alert) # the alert as this read shows it, not as the last answer did
1976
+ view.read_keyboard = tree.keyboard
1977
+ known = {e.ref for e in view.items} # what came back may get its old ref: look in both lists
1978
+ held = sorted([*view.items, *(e for e in view.shown if e.ref not in known)], key=lambda e: (e.y, e.x))
1979
+ # no action of the agent's came before this read: the content moved only if the elements say so
1980
+ view.items = changes.carry_refs(held, s.items, view.top_ref, moves_content=None)
1981
+ view.top_ref = max(view.top_ref, changes.highest(view.items))
1982
+ view.read_at, view.read_mark = self.control.clock(), mark # the next target in the same window needn't read again
1983
+ s.items = view.items
1984
+ return done.capped or mark[1] is None or not alike(self._mark()[1], mark[1])
1985
+
1986
+ @staticmethod
1987
+ def _missing(items: list[ui.Element], target: ui.Target) -> bool:
1988
+ """Does the target resolve to `not_found` (not `ambiguous`) on this list?"""
1989
+ try:
1990
+ ui.resolve(items, target)
1991
+ except Refused as e:
1992
+ return e.code == "not_found"
1993
+ return False
1994
+
1995
+ async def settle(self, after_seq: int, still: float | None = None, cap: float = CAP,
1996
+ since: Frame | None = None, early: int | None = None) -> tuple[float, bool]:
1997
+ """(seconds waited, changed): see _settle."""
1998
+ done = await self._settle(after_seq, still, cap, since, early)
1999
+ return done.seconds, done.changed
2000
+
2001
+ async def _settle(self, after_seq: int, still: float | None = None, cap: float = CAP,
2002
+ since: Frame | None = None, early: int | None = None) -> Settled:
2003
+ """Wait for the screen to be still, as the stream shows it. `after_seq` is the newest frame when the
2004
+ phone call returned: the settle ends only on a later frame, since earlier ones may predate the effect.
2005
+ WebDriverAgent streams continuously even when nothing changes, so a silence is a stall, not stillness.
2006
+
2007
+ `early` is the newest frame when the gesture went out. A tap's call returns about 200 ms after its effect
2008
+ shows, so the frames in between count too, but only once one of them is unlike `since`: they can then
2009
+ already hold the still run, and the settle ends at the first frame after the return that continues it.
2010
+
2011
+ Still: at least 2 frames alike (stream.alike) spanning `still` seconds (at least 2.5 median frame gaps,
2012
+ so a slow stream isn't called still too soon). Each frame is compared with the first of its run, so a
2013
+ slow fade, close to the last frame every time, is not still. `changed`: a frame is unlike the one at the
2014
+ start of the action (`since`, from _mark; None means there was no frame, so any frame counts).
2015
+ `changed` is False only when the answer is certain: 3 frames, all like `since`, and `quiet` seconds.
2016
+ A stall, no frames at all (it sleeps `settle_fallback` in total, first-frame wait included), too
2017
+ few frames, or the cap all report changed=True, so the tree gets read."""
2018
+ gap = self.hub.gap()
2019
+ still = max(self.settle_still if still is None else still, 2.5 * gap if gap else 0.0)
2020
+ stall = max(3 * gap if gap else 0.0, still)
2021
+ loop = asyncio.get_running_loop()
2022
+ began = loop.time()
2023
+ deadline = began + cap
2024
+ seq, differs, run, run_start, first_of_run = after_seq, False, 0, began, None
2025
+ if early is not None:
2026
+ last = None
2027
+ for arrived, frame in self.hub.since(early, upto=after_seq):
2028
+ differs = differs or not alike(frame, since)
2029
+ if alike(frame, first_of_run):
2030
+ run += 1
2031
+ else:
2032
+ run, run_start, first_of_run = 1, arrived, frame
2033
+ last = arrived
2034
+ if not differs: # nothing showed during the call: only frames after the return count
2035
+ run, run_start, first_of_run = 0, began, None
2036
+ elif (gap and last is not None and run >= 2 and began - run_start >= still
2037
+ and began - last <= 1.5 * gap):
2038
+ return Settled(0.0, True, False) # changed, then still up to a fresh frame at the return
2039
+ while (left := deadline - loop.time()) > 0:
2040
+ first = seq == after_seq
2041
+ wait = min(left, self.settle_first_frame if first else stall)
2042
+ frame = await self.hub.next_after(seq, timeout=wait)
2043
+ now = loop.time()
2044
+ if frame is None:
2045
+ if not first and wait == left:
2046
+ break # the cap came before the stall did
2047
+ if first: # no frames at all: the stream is down or still opening
2048
+ await asyncio.sleep(max(0.0, self.settle_fallback - (now - began)))
2049
+ return Settled(loop.time() - began, True, False) # or a stall
2050
+ seq = frame.seq
2051
+ differs = differs or not alike(frame, since)
2052
+ if alike(frame, first_of_run):
2053
+ run += 1
2054
+ else:
2055
+ run, run_start, first_of_run = 1, now, frame
2056
+ if run >= 2 and now - run_start >= still:
2057
+ if differs or (seq - after_seq >= 3 and now - began >= self.settle_quiet):
2058
+ return Settled(now - began, differs, False)
2059
+ return Settled(loop.time() - began, True, differs)
2060
+
2061
+ def _mark_read(self, view) -> None:
2062
+ """A tree read starts: note the stream's picture (its SHA-1 and, if it's recent, its bytes), so a
2063
+ screenshot shows the moment the list is of, not a later one."""
2064
+ view.read_mark = self._mark()
2065
+ frame = self.hub.latest
2066
+ fresh = frame is not None and self.hub.now_ms() - frame.at <= self.shot_fresh * 1000
2067
+ view.read_frame = frame if fresh else None
2068
+
2069
+ def _mark(self) -> tuple[int, Frame | None]:
2070
+ """(seq, the newest frame): where the stream stood at this moment."""
2071
+ frame = self.hub.latest
2072
+ return (frame.seq, frame) if frame else (0, None)
2073
+
2074
+ def _shot_args(self, args: dict, *, default: bool, custom: bool) -> tuple[int, bool] | None:
2075
+ """(width, labels) when a screenshot is wanted, else None. Only `look` takes width and labels;
2076
+ an action's screenshot uses look's defaults (AGENT-TOOLS §3.5)."""
2077
+ if not custom:
2078
+ return (shotlib.DEFAULT_WIDTH, False) if args.get("screenshot", default) else None
2079
+ width = args.get("width", shotlib.DEFAULT_WIDTH)
2080
+ if isinstance(width, bool) or not isinstance(width, (int, float)) or width != int(width):
2081
+ raise Refused("bad_params", "width must be a whole number of pixels")
2082
+ if not shotlib.MIN_WIDTH <= width <= shotlib.MAX_WIDTH:
2083
+ raise Refused("bad_params", f"width must be {shotlib.MIN_WIDTH}-{shotlib.MAX_WIDTH} pixels, not {width:g}")
2084
+ if not args.get("screenshot", default):
2085
+ return None
2086
+ return int(width), bool(args.get("labels", False))
2087
+
2088
+ async def _newest_frame(self) -> Frame | None:
2089
+ """The newest stream frame if it's recent, else the next one if it comes soon, else None."""
2090
+ latest = self.hub.latest
2091
+ if latest is not None and self.hub.now_ms() - latest.at <= self.shot_fresh * 1000:
2092
+ return latest
2093
+ return await self.hub.next_after(latest.seq if latest else 0, timeout=self.shot_wait)
2094
+
2095
+ async def _add_shot(self, answer: dict, view, shot: tuple[int, bool] | None, at_read: bool = True) -> dict:
2096
+ """Add the picture to an answer: from the stream (full resolution), else the phone's /screenshot.
2097
+ A picture that can't be had is a Note line, never an error. Decoding and resizing happen only here,
2098
+ so only when a picture was asked for. The picture is the frame from when the tree was read (`at_read`),
2099
+ else the newest one (the no-change shortcut read no tree); /screenshot only when there is no frame."""
2100
+ if shot is None:
2101
+ return answer
2102
+ width, labels = shot
2103
+ try:
2104
+ frame = view.read_frame if at_read else await self._newest_frame()
2105
+ try:
2106
+ source = await jpeg_of(frame) if frame is not None else None # a video picture is encoded here
2107
+ except PictureGone: # it was one of the older pictures, whose pixels are let go
2108
+ source = None
2109
+ turn = getattr(self.session, "turn", 0)
2110
+ if source is not None and turn and getattr(frame, "picture", None) is None:
2111
+ source = await self.turned(frame.seq, source, turn) # landscape: the stream's frame comes as captured
2112
+ if source is None:
2113
+ source = await self._read(self.session.screenshot_png)
2114
+ boxes = [(e.ref, *e.rect) for e in view.items if e.actions] if labels else None
2115
+ jpeg, size, skipped = await asyncio.to_thread(shotlib.render, source, self.session.screen, width, boxes)
2116
+ except Exception as e:
2117
+ answer["text"] += f"\nNote: no screenshot: {str(e) or type(e).__name__}"
2118
+ return answer
2119
+ if skipped:
2120
+ answer["text"] += "\nNote: labels skipped: the screen orientation changed."
2121
+ answer["image"], answer["image_size"] = base64.b64encode(jpeg).decode(), [size[0], size[1]]
2122
+ return answer
2123
+
2124
+ async def _look(self, view, shot: tuple[int, bool] | None, lines: list, notes: list) -> dict:
2125
+ async def read():
2126
+ self._mark_read(view)
2127
+ text, image = await self.session.look(screenshot=False)
2128
+ view.items = view.shown = self.session.items
2129
+ view.shown_off = self._off_screen_now()
2130
+ view.top_ref, view.app, view.seen = changes.highest(view.items), self.session.app, self.control.generation
2131
+ view.alert, view.keyboard = _alert_key(self.session.last_alert), self.session.last_keyboard
2132
+ view.read_keyboard = view.keyboard
2133
+ view.alert_now = view.alert
2134
+ view.read_at, view.moved, view.dropped = self.control.clock(), False, False
2135
+ view.unlooked = 0
2136
+ return text, image
2137
+ try:
2138
+ text, image = await self._read(read)
2139
+ except Exception as e:
2140
+ if not lines:
2141
+ raise
2142
+ view.moved = True # no list came back: the screen has probably changed
2143
+ return _answer(lines, [*notes, f"couldn't look afterwards: {e}."])
2144
+ return await self._add_shot(_answer([*lines, text], notes, image), view, shot)
2145
+
2146
+ async def _after(self, view, result: str, notes: list, full: bool, moves_content: bool | None = False,
2147
+ seconds: float | None = None, snap: Snapshot | None = None, drifted: bool = False) -> dict:
2148
+ """An action's answer: its result line, then what changed on screen since the agent's last answer
2149
+ (`view.shown`), or the full list when the app changed or `full`, then `Settled in` (left out when
2150
+ `seconds` is None: `wait` did nothing to settle), then the notes. Refs follow their elements."""
2151
+ s = self.session
2152
+ tail = [] if seconds is None else [settled_line(seconds)]
2153
+ app_before, listed_before = view.app, view.shown
2154
+
2155
+ async def read():
2156
+ self._mark_read(view)
2157
+ read_now = snap or await s.snapshot() # `snap`: a read the caller already made (wait_for's last poll)
2158
+ s.items = read_now.tree.items
2159
+ app_changed = read_now.app is not None and view.app is not None and read_now.app != view.app
2160
+ head = _furniture(view, read_now.tree.alert, read_now.tree.keyboard)
2161
+ up = changes.update(view.items, read_now.tree.items, app_changed, view.top_ref, view.shown, full, head,
2162
+ moves_content)
2163
+ return read_now, up
2164
+ try:
2165
+ snap, up = await self._read(read)
2166
+ except Exception as e:
2167
+ view.moved = True # no list came back: the screen has probably changed
2168
+ return _answer([result, *tail], [*notes, f"couldn't look afterwards: {e}."])
2169
+ covered = not snap.tree.items and bool(view.shown) # the agent saw elements; now there are none
2170
+ s.items = view.items = view.shown = up.items
2171
+ view.top_ref, view.app = up.top, snap.app or view.app
2172
+ view.alert, view.keyboard = _alert_key(snap.tree.alert), snap.tree.keyboard
2173
+ view.read_keyboard = view.keyboard
2174
+ view.alert_now = view.alert
2175
+ view.seen, view.read_at, view.moved, view.dropped = (self.control.generation, self.control.clock(),
2176
+ False, False)
2177
+ # what changed off screen (v1.6), against what lay there at the agent's last answer
2178
+ off_was, view.shown_off = view.shown_off, self._off_screen_of(snap.tree)
2179
+ # ...unless the old list is no baseline: another app is in front now, or the full list was forced (a
2180
+ # person acted, the line dropped, the agent had never looked). A place only says "the same element" when
2181
+ # nothing can have moved the content since the agent's last answer.
2182
+ same_app = not (snap.app is not None and app_before is not None and snap.app != app_before)
2183
+ still = not (moves_content is not False or drifted or view.unlooked)
2184
+ off = (changes.off_screen(off_was, view.shown_off, by_place=still, listed=listed_before)
2185
+ if same_app and not full else [])
2186
+ if off and not up.full:
2187
+ up.lines = [line for line in up.lines if line != changes.NO_CHANGE] + off
2188
+ earlier, view.unlooked = view.unlooked, 0
2189
+ if earlier and not up.full:
2190
+ # the block is measured from the agent's last list, so it holds those actions' changes too: a
2191
+ # first-time agent closed a sheet it had opened with look: false and read "No visible change."
2192
+ notes = [*notes, NOTE_SPAN.format(n=earlier + 1)]
2193
+ if up.full:
2194
+ tree = snap.tree
2195
+ head = ui.describe(up.items, s.screen, snap.scale, snap.app, tree.alert, tree.keyboard, tree.more,
2196
+ (tree.above, tree.below))
2197
+ # the full list because most of the screen changed: what changed off screen still follows it, under
2198
+ # a heading of its own (bare `~` lines after a list read as a second, broken block)
2199
+ beyond = [changes.OFF_SCREEN, *off] if off else []
2200
+ return _answer([result, head, *beyond, *tail], [*notes, *([NOTE_COVERED] if covered else [])])
2201
+ return _answer([result, *up.lines, *tail], notes)
2202
+
2203
+ def _off_screen_now(self) -> list[ui.Element]:
2204
+ """What the session's last read found off screen, without what an open sheet hides (as in the header's
2205
+ counts). Only right inside the read itself: another client's read may come next."""
2206
+ s = self.session
2207
+ hidden = {id(e) for e, _ in getattr(s, "behind", [])}
2208
+ return [e for e in getattr(s, "offscreen", []) if id(e) not in hidden]
2209
+
2210
+ def _off_screen_of(self, tree: ui.Tree) -> list[ui.Element]:
2211
+ """The same from one read's own tree. (A tree a test made by hand has no off-screen list of its own: the
2212
+ session's stands in.)"""
2213
+ if not tree.offscreen and not tree.behind:
2214
+ return self._off_screen_now()
2215
+ hidden = {id(e) for e, _ in tree.behind}
2216
+ return [e for e in tree.offscreen if id(e) not in hidden]
2217
+
2218
+ def _current(self, view) -> bool:
2219
+ """Is the agent's last list still the screen? Not if a person or another agent acted since
2220
+ (a different generation), if an action returned no list since, or if it is older than
2221
+ LIVE_AGE seconds and the screen may have changed meanwhile."""
2222
+ if view.seen != self.control.generation or view.moved or view.dropped:
2223
+ return False
2224
+ return self.control.clock() - view.read_at < LIVE_AGE or self._screen_unchanged(view)
2225
+
2226
+ def _screen_unchanged(self, view) -> bool:
2227
+ """Does the stream show the picture the list was read against? Only a fresh newest frame that
2228
+ matches the one at read time says so; with no stream the answer is no, and the list is read again."""
2229
+ frame, (_, was) = self.hub.latest, view.read_mark
2230
+ if frame is None or was is None or self.hub.now_ms() - frame.at > FRESH_FRAME * 1000:
2231
+ return False
2232
+ return alike(frame, was)
2233
+
2234
+ async def _read(self, fn):
2235
+ """A read through the queue; after a dropped line, once more when reconnected."""
2236
+ for attempt in (1, 2):
2237
+ try:
2238
+ return await self.control.read(lambda: self._connected_only(fn))
2239
+ except ConnectionLost as e:
2240
+ self.lost(e)
2241
+ if attempt == 2:
2242
+ raise Refused("reconnecting", "lost the connection to the phone; try again") from None
2243
+ await self.wait_connected()
2244
+
2245
+ # ---- a person, from the live view ----
2246
+ async def viewer_message(self, message: dict) -> dict | None:
2247
+ """One message from a live view (docs/VIEWER-PROTOCOL.md). Returns an error message for
2248
+ that viewer, or None. Unknown types are ignored."""
2249
+ kind = message.get("type")
2250
+ try:
2251
+ if kind == "pause":
2252
+ self.control.set_paused(bool(message.get("on")))
2253
+ self.broadcast(self.status())
2254
+ return None
2255
+ if kind == "quality":
2256
+ await self.set_quality(str(message.get("preset")))
2257
+ return None
2258
+ if kind not in ("gesture", "scroll", "text", "key", "button"):
2259
+ return None # "overlay" comes in part 2; anything else is unknown
2260
+ if self.state != "live":
2261
+ return {"type": "error", "code": {"locked": "locked", "not-answering": "not_answering"}.get(self.state, "reconnecting"),
2262
+ "message": f"not connected to the phone ({self.detail or self.state})"}
2263
+ run, action, point = self._person_action(kind, message)
2264
+ self._person_at = time.monotonic()
2265
+
2266
+ async def quickened():
2267
+ await self._quicken() # so the picture after the input comes at full rate (V1)
2268
+ return await run()
2269
+ await self.control.person(lambda: self._connected_only(quickened))
2270
+ self._announce("person", action, point=self._point(point))
2271
+ return None
2272
+ except Refused as e:
2273
+ return {"type": "error", "code": e.code, "message": str(e)}
2274
+ except ConnectionLost as e:
2275
+ self.lost(e)
2276
+ return {"type": "error", "code": "reconnecting",
2277
+ "message": "the connection dropped; your input may not have reached the phone"}
2278
+ except (ValueError, KeyError, TypeError, IndexError) as e:
2279
+ return {"type": "error", "code": "unsupported", "message": f"bad {kind} message: {e}"}
2280
+ except Exception as e:
2281
+ return {"type": "error", "message": str(e)[:300]}
2282
+
2283
+ def _person_action(self, kind: str, m: dict):
2284
+ """(coroutine function to run, kind for the action message, start point in points)."""
2285
+ s = self.session
2286
+ if kind == "gesture":
2287
+ g = classify(m["points"], s.screen)
2288
+ if g.kind == "drag":
2289
+ return (lambda: s.path(list(g.points))), "drag", g.points[0][1:]
2290
+ _, x, y = g.points[0]
2291
+ return (lambda: s.tap(ui.Target(x=x, y=y), hold_ms=g.hold_ms)), g.kind, (x, y)
2292
+ if kind == "scroll":
2293
+ x0, y0, x1, y1 = ui.scroll_swipe(float(m["x"]), float(m["y"]), float(m.get("dx", 0)),
2294
+ float(m.get("dy", 0)), s.screen)
2295
+ return (lambda: s.scroll(x0, y0, x1, y1)), "scroll", (x0, y0) # a wheel stops where it should
2296
+ if kind == "text":
2297
+ text = str(m["text"])
2298
+ if not text:
2299
+ raise ValueError("no text")
2300
+ return (lambda: s.type_text(text)), "text", None
2301
+ if kind == "key":
2302
+ name = str(m["key"])
2303
+ if name not in ("backspace", "enter"):
2304
+ raise ValueError(f"unknown key {name!r}")
2305
+ return (lambda: s.key(name)), name, None
2306
+ name = str(m["name"])
2307
+ if name not in BUTTONS:
2308
+ raise ValueError(f"unknown button {name!r}")
2309
+ return (lambda: s.press_button(name)), name, None
2310
+
2311
+ async def set_quality(self, preset: str) -> None:
2312
+ if preset not in PRESETS:
2313
+ raise ValueError(f"preset must be one of {', '.join(PRESETS)}")
2314
+ if self.state in ("live", "locked"):
2315
+ await self._read(lambda: self.session.settings({**PRESETS[preset], **NO_IDLE_WAIT}))
2316
+ self._fps = int(PRESETS[preset]["mjpegServerFramerate"]) # full rate; the pace slows it again when still
2317
+ self.preset = preset
2318
+ if self.video is not None:
2319
+ self.video.set_quality(preset)
2320
+ self.broadcast(self.status())
2321
+
2322
+ # ---- the stream and shutting down ----
2323
+ async def _open_stream(self):
2324
+ try:
2325
+ await asyncio.wait_for(self._connected.wait(), 30)
2326
+ except asyncio.TimeoutError:
2327
+ raise ConnectionLost("not connected to the phone yet") from None
2328
+ return await self.session.device.open_stream()
2329
+
2330
+ async def start_video(self, tries: int = 3, gap: float = 3.0) -> str:
2331
+ """Set up the H.264 live view: the hub, the AirPlay receiver, and Screen Mirroring on the phone, started
2332
+ the way the owner would (mirror.py). The app that was in front comes back afterwards. Returns what
2333
+ happened in words; it never raises, since the JPEG stream stays as the picture."""
2334
+ from . import airplay, mirror, video
2335
+ s = self.session
2336
+ if not video.available():
2337
+ return self._video_note("PyAV isn't installed (pip install tapwright[video])")
2338
+ uxplay = airplay.find()
2339
+ if uxplay is None:
2340
+ return self._video_note("no AirPlay receiver found (install UxPlay, or set TAPWRIGHT_UXPLAY)")
2341
+ try:
2342
+ scale = s.scale or 3.0
2343
+ pixels = (round(s.screen[0] * scale), round(s.screen[1] * scale))
2344
+ receiver = airplay.Receiver(uxplay, 0, pixels, self.log)
2345
+ hub = video.VideoHub(port=0, screen_width=lambda: s.screen[0], mirroring=lambda: receiver.mirroring,
2346
+ log=self.log)
2347
+ receiver.args = airplay.command(uxplay, hub.port, pixels)
2348
+ receiver.start()
2349
+ self.video, self._receiver = hub, receiver
2350
+ if self.settle_on_video:
2351
+ self._start_feeder()
2352
+ hub.set_quality(self.preset)
2353
+ said = ""
2354
+ for attempt in range(tries):
2355
+ await asyncio.sleep(gap) # the receiver takes a moment to show up in the phone's list
2356
+ async with self.control.exclusive():
2357
+ self._require_connected()
2358
+ front = await s.foreground_bundle_id()
2359
+ try:
2360
+ said = await mirror.start(s, airplay.NAME)
2361
+ except mirror.MirrorError as e:
2362
+ said = str(e)
2363
+ continue
2364
+ finally:
2365
+ if front and front != "com.apple.springboard":
2366
+ await s.launch(front) # back to the app that was in front
2367
+ break
2368
+ for view in self.control.clients.values():
2369
+ view.dropped = True # the phone left the app for a moment: every agent's list is old
2370
+ self.broadcast(self.status())
2371
+ return self._video_note(said, ok=receiver.mirroring or said.startswith(("Mirroring", "Already")))
2372
+ except Exception as e: # the JPEG stream stays: video is an extra
2373
+ return self._video_note(f"couldn't start: {e}")
2374
+
2375
+ def _start_feeder(self) -> None:
2376
+ """While `video` is live its pictures feed `hub` (and the MJPEG stream stays closed); when it isn't, the
2377
+ MJPEG stream does, as before. The feeder checks which, so mirroring may stop and start again."""
2378
+ if self._feeder_task is None and self.video is not None:
2379
+ self._feeder = PictureFeeder(self.video, self.hub, log=self.log)
2380
+ self._feeder_task = asyncio.create_task(self._feeder.run(), name="feeder")
2381
+
2382
+ def _video_note(self, text: str, ok: bool = False) -> str:
2383
+ self.log(f"video: {text}" if ok else f"video: {text}; the live view stays on the JPEG stream")
2384
+ return text
2385
+
2386
+ async def close(self) -> None:
2387
+ if self._feeder_task is not None:
2388
+ self._feeder_task.cancel()
2389
+ await asyncio.gather(self._feeder_task, return_exceptions=True)
2390
+ await self.pump.stop()
2391
+ self.hub.close()
2392
+ if self._video_task is not None:
2393
+ self._video_task.cancel()
2394
+ if self._receiver is not None:
2395
+ await asyncio.to_thread(self._receiver.stop)
2396
+ if self.video is not None:
2397
+ await asyncio.to_thread(self.video.close) # joins its threads
2398
+ try:
2399
+ await asyncio.wait_for(self.session.close(), 10)
2400
+ except Exception as e:
2401
+ self.log(f"closing: {e}")