tapwright 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tapwright/__init__.py +2 -0
- tapwright/__main__.py +5 -0
- tapwright/airplay.py +103 -0
- tapwright/changes.py +264 -0
- tapwright/cli.py +374 -0
- tapwright/client.py +151 -0
- tapwright/control.py +189 -0
- tapwright/daemon.py +2401 -0
- tapwright/descriptions.py +186 -0
- tapwright/device.py +495 -0
- tapwright/errors.py +13 -0
- tapwright/mcp_server.py +266 -0
- tapwright/mirror.py +91 -0
- tapwright/rtp.py +109 -0
- tapwright/server.py +435 -0
- tapwright/session.py +672 -0
- tapwright/shot.py +42 -0
- tapwright/signing.py +201 -0
- tapwright/skill/SKILL.md +203 -0
- tapwright/skill/agents/openai.yaml +4 -0
- tapwright/skill/references/setup.md +102 -0
- tapwright/skills.py +60 -0
- tapwright/store.py +130 -0
- tapwright/stream.py +405 -0
- tapwright/tapline.py +118 -0
- tapwright/ui.py +1242 -0
- tapwright/video.py +486 -0
- tapwright/viewer.html +2060 -0
- tapwright-1.0.0.dist-info/METADATA +254 -0
- tapwright-1.0.0.dist-info/RECORD +34 -0
- tapwright-1.0.0.dist-info/WHEEL +5 -0
- tapwright-1.0.0.dist-info/entry_points.txt +2 -0
- tapwright-1.0.0.dist-info/licenses/LICENSE +21 -0
- tapwright-1.0.0.dist-info/top_level.txt +1 -0
tapwright/daemon.py
ADDED
|
@@ -0,0 +1,2401 @@
|
|
|
1
|
+
"""The phone side of `tapwright serve`: one Session (tunnel and WebDriverAgent), the control
|
|
2
|
+
rules, the stream, the agent tools, a person's input from the live view, and a supervisor
|
|
3
|
+
that keeps it all connected. server.py only turns HTTP into calls here."""
|
|
4
|
+
import asyncio
|
|
5
|
+
import base64
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import dataclasses
|
|
9
|
+
import math
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import time
|
|
13
|
+
import traceback
|
|
14
|
+
from contextlib import nullcontext
|
|
15
|
+
from typing import NamedTuple
|
|
16
|
+
|
|
17
|
+
from PIL import Image, UnidentifiedImageError
|
|
18
|
+
|
|
19
|
+
from . import changes, ui
|
|
20
|
+
from . import session as session_module
|
|
21
|
+
from . import shot as shotlib
|
|
22
|
+
from .control import TAP_MS, Controller, Refused, classify
|
|
23
|
+
from .device import (AppNotAnswering, ConnectionLost, DeviceError, NotSetUp, PasscodeNeeded, PhoneLocked, Unanswered,
|
|
24
|
+
PhoneNotFound)
|
|
25
|
+
from .session import Snapshot
|
|
26
|
+
from .tapline import AgentTouches
|
|
27
|
+
from .stream import (NO_IDLE_WAIT, PRESETS, Frame, FrameHub, PictureFeeder, PictureGone, StreamPump, alike, jpeg_of,
|
|
28
|
+
turn_jpeg)
|
|
29
|
+
|
|
30
|
+
PROTOCOL = 1
|
|
31
|
+
ACCEPTS = ["tap", "longPress", "drag", "scroll", "text", "enter", "backspace", "home", "volume", "quality"]
|
|
32
|
+
BUTTONS = ("home", "volumeUp", "volumeDown")
|
|
33
|
+
STILL = 0.1 # seconds without a changed frame before a screen counts as still (docs/AGENT-TOOLS.md §6)
|
|
34
|
+
RESOLVE_SETTLE = 1.0 # the longest a silent re-read waits for the screen to be still
|
|
35
|
+
CAP = 2.0 # the longest an action's answer waits for the screen
|
|
36
|
+
LONG_CAP = 4.0 # the same after launching an app or opening a URL
|
|
37
|
+
LIVE_AGE = 2.0 # seconds a list stays current for id, label and text targets (docs/AGENT-TOOLS.md §2)
|
|
38
|
+
FRESH_FRAME = 2.0 # seconds a stream frame counts as "the screen now"
|
|
39
|
+
MIN_QUIET = 0.3 # seconds a screen must stay as it was before "No visible change." is believed
|
|
40
|
+
NOTE_CAPPED = ("the screen was still changing after {cap:g} s (an animation or video); "
|
|
41
|
+
"positions may be mid-change.")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Scope(NamedTuple):
|
|
45
|
+
"""Personal mode: the apps an agent may read and touch, as bundle ids or exact names. Empty: none, so
|
|
46
|
+
nothing runs (it fails closed). A dedicated device has no Scope at all."""
|
|
47
|
+
apps: tuple[str, ...] = ()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
NO_APPS = ("no_apps: this agent was started without an app list, so it may not use the phone. Start it with "
|
|
51
|
+
"`tapwright mcp --apps <bundle id>` for the apps under test, or with `--dedicated` on a device "
|
|
52
|
+
"kept for testing.")
|
|
53
|
+
SCOPE_FREE = ("open_app", "close_app", "open_url", "press_button") # how an agent gets back to its app
|
|
54
|
+
APP_LINE = re.compile(r"^App: (.+) \(([^()\s]+)\)$", re.M) # the header line of a full list: name, bundle id
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _outside(bundle: str | None, name: str | None, given: str) -> str:
|
|
58
|
+
front = "the phone can't say which app" if not bundle else f"{name} ({bundle})" if name else bundle
|
|
59
|
+
return (f"outside_apps: {front} is in front and isn't one of the apps you were given ({given}), so nothing of it is shown or "
|
|
60
|
+
"touched. Go back with open_app.")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Settled(NamedTuple):
|
|
64
|
+
seconds: float
|
|
65
|
+
changed: bool # False only when the stream showed the screen unchanged for certain
|
|
66
|
+
capped: bool # the cap ended the wait with frames still changing
|
|
67
|
+
WAIT_MIN, WAIT_MAX = 0.1, 30.0 # seconds, for `wait`
|
|
68
|
+
WAIT_FOR_MIN, WAIT_FOR_DEFAULT = 0.5, 10.0 # `wait_for`'s timeout in seconds (its upper end is WAIT_MAX)
|
|
69
|
+
WAIT_FOR_KINDS = ("text", "id", "label")
|
|
70
|
+
WAIT_FOR_POLL = 0.3 # seconds between tree reads while waiting
|
|
71
|
+
SCROLL_MIN, SCROLL_MAX = 0.1, 1.0 # `scroll`'s amount: a share of the scroll area
|
|
72
|
+
MAX_SCROLL_SWIPES = 15 # `scroll to start|end` gives up after this many swipes
|
|
73
|
+
SCROLL_CAP = 12.0 # ... or after this many seconds (a spinner or a bounce can keep the screen moving)
|
|
74
|
+
SCROLL_TO = {"end": ("down", "right"), "start": ("up", "left")} # the directions each `to` can mean; the first is the default
|
|
75
|
+
SCROLL_TOP_INSET, SCROLL_BOTTOM_INSET = 60.0, 40.0 # points a swipe keeps off the screen's edges when no bar says more
|
|
76
|
+
WAIT_FOR_READ_MAX = 5.0 # the longest one tree read in `wait_for` may take
|
|
77
|
+
WAIT_FOR_MIN_LEFT = 0.1 # with less time than this left, `wait_for` gives up instead of reading again
|
|
78
|
+
WAIT_FOR_FIRST_READ = 3.0 # `wait_for`'s first read of the screen gets at least this long (twice the slowest look allowed)
|
|
79
|
+
# The stream's pace. Every frame WebDriverAgent captures costs the phone, mostly in iOS's testmanagerd: about 88% of
|
|
80
|
+
# a core at balanced's ~27 fps and 38% at light's ~10 fps (DVT sysmontap, the lab animating, 2026-11-10). So the
|
|
81
|
+
# stream runs at its preset's rate while something happens, and at STILL_FPS once nothing has: no agent call for
|
|
82
|
+
# the first quiet span (the stream's own linger, so an agent between steps never waits for the rate to rise), no
|
|
83
|
+
# person's input for the second, no change on the screen for the third. STILL_FPS keeps the longest gap
|
|
84
|
+
# between frames at 0.2 s (V3: at most 0.25 s).
|
|
85
|
+
STILL_FPS = 5
|
|
86
|
+
PACE_QUIET = (30.0, 2.0, 1.0) # seconds without an agent call, a person's input, a change on the screen
|
|
87
|
+
PACE_TICK = 0.25 # how often the pace is checked
|
|
88
|
+
LOOP_TICK, LOOP_HELD, LOOP_SAY_EVERY = 0.05, 0.3, 10.0 # seconds: see Daemon._watch_loop
|
|
89
|
+
BACK_STILL_PAUSE, BACK_STILL_TRIES = 0.3, 6 # see Daemon._still_back
|
|
90
|
+
TOGGLE_LAND_WAIT, TOGGLE_POLL = 0.5, 0.05 # seconds: see Daemon._toggle_landed
|
|
91
|
+
# A switch element this many times wider than tall is a whole row (label and control): toggle touches the control,
|
|
92
|
+
# SWITCH_FROM_RIGHT points in from its right end (half an iOS switch, 63 pt wide), not the label in the middle
|
|
93
|
+
WIDE_SWITCH, SWITCH_FROM_RIGHT = 3.0, 32.0
|
|
94
|
+
NOTE_PERSON = "a person used the phone since your last look."
|
|
95
|
+
# a notification banner took a touch near the top (Daemon._taken_away)
|
|
96
|
+
NOTE_TAKEN = ("{other} came to the front after the touch: the app opened it, or a notification banner over the top of "
|
|
97
|
+
"the screen took the touch. Went back to {app}. If the app was meant to open it, that worked; if not, "
|
|
98
|
+
"look, then try again.")
|
|
99
|
+
NOTE_TAKEN_SYSTEM = ("{other} came to the front after the touch: a notification banner over the top of the screen, or "
|
|
100
|
+
"a drag from its top edge, can take a touch there. Went back to {app}; the touch may not have "
|
|
101
|
+
"reached it. Look, then try again.")
|
|
102
|
+
SYSTEM_IN_FRONT = "the home screen, Notification Center or Control Center"
|
|
103
|
+
RECHECK_WITHIN = 5.0 # seconds: a banner's theft shows within about one; later, the agent or a person moved on
|
|
104
|
+
NOTE_KEYBOARD = "closed the keyboard first."
|
|
105
|
+
NOTE_RETYPED = "the field didn't hold what was typed (keys went missing), so it was cleared and typed once more."
|
|
106
|
+
SCROLL_HINT = 'scroll to it first (scroll with this target and to: "target").' # AGENT-TOOLS v1.6: the one-call way
|
|
107
|
+
NOTE_SPAN = "changes are since your last list, {n} actions ago." # after answers with look: false (v1.6)
|
|
108
|
+
# v1.6: `force` let through what the checks would have refused. A real touch doesn't make the hidden element act
|
|
109
|
+
NOTE_FORCED = "forced ({reason}): the touch may have landed on what is in front."
|
|
110
|
+
KEYBOARD_TOOLS = ("tap", "long_press", "toggle", "type_text") # the touches that close the keyboard before them
|
|
111
|
+
NOTE_COVERED = "something outside the app (a system sheet or browser) may cover it."
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _size(jpeg: bytes | None) -> list[int] | None:
|
|
115
|
+
"""[width, height] in pixels of a JPEG, or None when there is no readable picture."""
|
|
116
|
+
if not jpeg:
|
|
117
|
+
return None
|
|
118
|
+
try:
|
|
119
|
+
with Image.open(io.BytesIO(jpeg)) as img:
|
|
120
|
+
return [img.width, img.height]
|
|
121
|
+
except (UnidentifiedImageError, OSError):
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
MARK_NAME_MAX = 32 # characters of an element's name in the live view (VIEWER-PROTOCOL 1.4)
|
|
126
|
+
CAPTION_MAX = 60 # characters of a step's caption (VIEWER-PROTOCOL 1.4)
|
|
127
|
+
BUTTON_CAPTIONS = {"home": "Home", "volumeUp": "Volume up", "volumeDown": "Volume down"}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def element_name(result: str) -> str:
|
|
131
|
+
"""The element's name in an action's first answer line ("Tapped e25 button "Equals" @320,780 95x90." →
|
|
132
|
+
`Equals`), cut to MARK_NAME_MAX; "" when it names none. The live view's `label` (VIEWER-PROTOCOL 1.4): the
|
|
133
|
+
whole answer line covered the keys around a touch and ran off the picture (recording the demo, 2026-11-11)."""
|
|
134
|
+
name = re.search(r'"([^"]+)"', result)
|
|
135
|
+
if name is None:
|
|
136
|
+
return ""
|
|
137
|
+
text = name.group(1)
|
|
138
|
+
return text if len(text) <= MARK_NAME_MAX else text[:MARK_NAME_MAX - 1] + "…"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def step_caption(tool: str, result: str, args: dict) -> str | None:
|
|
142
|
+
"""The agent's step in plain words for people watching the live view (VIEWER-PROTOCOL 1.4, "Captions"):
|
|
143
|
+
`Tap "Sign up"`, `Type into "Email"`, `Scroll down`, `Open Wayfare`. Never the typed text, a link's address,
|
|
144
|
+
refs or coordinates. None for a step a viewer needn't see."""
|
|
145
|
+
name = element_name(result)
|
|
146
|
+
quoted = f'"{name}"' if name else ""
|
|
147
|
+
if tool in ("tap", "double_tap"):
|
|
148
|
+
text = f"Tap {quoted}"
|
|
149
|
+
elif tool == "long_press":
|
|
150
|
+
text = f"Press and hold {quoted}"
|
|
151
|
+
elif tool == "type_text":
|
|
152
|
+
text = f"Type into {quoted}" if name else "Type"
|
|
153
|
+
elif tool == "scroll":
|
|
154
|
+
to = args.get("to")
|
|
155
|
+
if to == "target" and name:
|
|
156
|
+
text = f"Scroll to {quoted}"
|
|
157
|
+
else:
|
|
158
|
+
text = f"Scroll {args.get('direction') or {'end': 'down', 'start': 'up'}.get(to or '', '')}"
|
|
159
|
+
elif tool == "swipe":
|
|
160
|
+
text = "Swipe"
|
|
161
|
+
elif tool == "toggle":
|
|
162
|
+
text = f"Switch {quoted}"
|
|
163
|
+
elif tool == "back":
|
|
164
|
+
text = "Go back"
|
|
165
|
+
elif tool == "alert":
|
|
166
|
+
text = "Answer the alert"
|
|
167
|
+
elif tool == "open_url":
|
|
168
|
+
text = "Open link" # never the address: a link can carry codes or tokens
|
|
169
|
+
elif tool in ("open_app", "close_app"):
|
|
170
|
+
app = re.match(r"(?:Opened|Closed) (.+?)(?: \(|\.|,|$)", result)
|
|
171
|
+
text = f"{'Open' if tool == 'open_app' else 'Close'} {app.group(1) if app else 'the app'}"
|
|
172
|
+
elif tool == "rotate":
|
|
173
|
+
text = "Turn the phone"
|
|
174
|
+
elif tool == "press_button":
|
|
175
|
+
text = BUTTON_CAPTIONS.get(str(args.get("name") or ""), "Press a button")
|
|
176
|
+
elif tool == "wait_for":
|
|
177
|
+
return None
|
|
178
|
+
else:
|
|
179
|
+
text = tool.replace("_", " ")
|
|
180
|
+
text = " ".join(text.split())
|
|
181
|
+
return text if len(text) <= CAPTION_MAX else text[:CAPTION_MAX - 1] + "…"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _answer(lines: list[str], notes: list[str] | None = None, image: bytes | None = None) -> dict:
|
|
185
|
+
"""One tool answer (docs/AGENT-TOOLS.md §3): the lines as one text, a `Note: ` line per note,
|
|
186
|
+
and the picture, if any, with its size."""
|
|
187
|
+
text = "\n".join([*lines, *(f"Note: {n}" for n in notes or [])])
|
|
188
|
+
return {"text": text, "image": base64.b64encode(image).decode() if image else None, "image_size": _size(image)}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def settled_line(seconds: float) -> str:
|
|
192
|
+
return f"Settled in {seconds:.2f} s."
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _alert_key(alert: ui.Alert | None) -> tuple[str, tuple[str, ...]] | None:
|
|
196
|
+
return (alert.title, alert.buttons) if alert else None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _furniture(view, alert: ui.Alert | None, keyboard: bool) -> list[str]:
|
|
200
|
+
"""What an alert or the keyboard did since the agent's last answer, as the first lines of a change block."""
|
|
201
|
+
lines = []
|
|
202
|
+
if _alert_key(alert) != view.alert:
|
|
203
|
+
lines.append(ui.alert_line(alert) if alert else "Alert: gone")
|
|
204
|
+
if keyboard != view.keyboard:
|
|
205
|
+
lines.append(f"Keyboard: {'shown' if keyboard else 'hidden'}")
|
|
206
|
+
return lines
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _number(args: dict, key: str) -> float | None:
|
|
210
|
+
"""args[key] as a finite float; None when it isn't given. Anything else is bad_params."""
|
|
211
|
+
value = args.get(key)
|
|
212
|
+
if value is None:
|
|
213
|
+
return None
|
|
214
|
+
try:
|
|
215
|
+
number = float(value)
|
|
216
|
+
except (TypeError, ValueError):
|
|
217
|
+
number = math.nan
|
|
218
|
+
if not math.isfinite(number):
|
|
219
|
+
raise Refused("bad_params", f"{key} must be a number, not {value!r}")
|
|
220
|
+
return number
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _checked(name: str, args: dict) -> dict:
|
|
224
|
+
"""The agent's arguments with numbers coerced, or Refused(bad_params) naming the one that's wrong.
|
|
225
|
+
This runs before anything reaches the phone, so a ValueError later is never the agent's fault."""
|
|
226
|
+
checked = dict(args)
|
|
227
|
+
for key in ("x0", "y0", "x1", "y1", "seconds", "amount"):
|
|
228
|
+
if (number := _number(args, key)) is not None:
|
|
229
|
+
checked[key] = number
|
|
230
|
+
if args.get("target") is not None:
|
|
231
|
+
checked["target"] = ui.parse_target(args["target"]) # bad_params, with an example of a target
|
|
232
|
+
if name == "scroll":
|
|
233
|
+
if checked.get("direction") is None: # null (the MCP server sends every argument) is left out
|
|
234
|
+
checked.pop("direction", None)
|
|
235
|
+
if checked.get("direction", "down") not in ("up", "down", "left", "right"):
|
|
236
|
+
raise Refused("bad_params", f"direction must be up, down, left or right, not {args['direction']!r}")
|
|
237
|
+
if args.get("to") not in (None, "start", "end", "target"):
|
|
238
|
+
raise Refused("bad_params", f"to must be start, end or target, not {args['to']!r}")
|
|
239
|
+
if args.get("to") == "target":
|
|
240
|
+
aim = checked.get("target")
|
|
241
|
+
if aim is None or aim.point is not None:
|
|
242
|
+
raise Refused("bad_params", 'to: "target" scrolls to an element: pass it as target (ref, id, label '
|
|
243
|
+
"or text)")
|
|
244
|
+
if "direction" in checked:
|
|
245
|
+
raise Refused("bad_params", 'to: "target" finds the direction itself; leave direction out')
|
|
246
|
+
elif args.get("to") and "direction" in checked and checked["direction"] not in SCROLL_TO[args["to"]]:
|
|
247
|
+
raise Refused("bad_params", f"to {args['to']} scrolls {' or '.join(SCROLL_TO[args['to']])}, not "
|
|
248
|
+
f"{args['direction']}; leave direction out, or make them agree")
|
|
249
|
+
if "amount" in checked and not SCROLL_MIN <= checked["amount"] <= SCROLL_MAX:
|
|
250
|
+
raise Refused("bad_params", f"amount must be from {SCROLL_MIN:g} to {SCROLL_MAX:g}, not {args['amount']!r}")
|
|
251
|
+
if name == "type_text":
|
|
252
|
+
if not isinstance(args.get("text"), str):
|
|
253
|
+
raise Refused("bad_params", f"text must be a string, not {args.get('text')!r}")
|
|
254
|
+
for key in ("replace", "submit", "force"):
|
|
255
|
+
if key in args and not isinstance(args[key], bool):
|
|
256
|
+
raise Refused("bad_params", f"{key} must be true or false, not {args[key]!r}")
|
|
257
|
+
if name == "rotate" and args.get("to") not in ("portrait", "landscape_left", "landscape_right"):
|
|
258
|
+
raise Refused("bad_params", f"to must be portrait, landscape_left or landscape_right, not {args.get('to')!r}")
|
|
259
|
+
if name == "wait_for":
|
|
260
|
+
given = [k for k in WAIT_FOR_KINDS if args.get(k) is not None]
|
|
261
|
+
if len(given) != 1:
|
|
262
|
+
raise Refused("bad_params", "wait_for needs exactly one of text, id or label")
|
|
263
|
+
if not isinstance(args[given[0]], str) or not args[given[0]]:
|
|
264
|
+
raise Refused("bad_params", f"{given[0]} must be a non-empty string, not {args[given[0]]!r}")
|
|
265
|
+
if "gone" in args and not isinstance(args["gone"], bool):
|
|
266
|
+
raise Refused("bad_params", f"gone must be true or false, not {args['gone']!r}")
|
|
267
|
+
timeout = args.get("timeout", WAIT_FOR_DEFAULT)
|
|
268
|
+
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not WAIT_FOR_MIN <= timeout <= WAIT_MAX:
|
|
269
|
+
raise Refused("bad_params", f"timeout must be a number from {WAIT_FOR_MIN:g} to {WAIT_MAX:g}, not {timeout!r}")
|
|
270
|
+
for key in ("look", "screenshot", "labels"):
|
|
271
|
+
if key in args and not isinstance(args[key], bool):
|
|
272
|
+
raise Refused("bad_params", f"{key} must be true or false, not {args[key]!r}")
|
|
273
|
+
if name == "wait":
|
|
274
|
+
seconds = args.get("seconds") # required (REQUIRED); a bool is a number to Python, a string isn't one to us
|
|
275
|
+
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or not WAIT_MIN <= seconds <= WAIT_MAX:
|
|
276
|
+
raise Refused("bad_params", f"seconds must be a number from {WAIT_MIN:g} to {WAIT_MAX:g}, not {seconds!r}")
|
|
277
|
+
if name in ("open_app", "close_app") and (not isinstance(args["app"], str) or not args["app"].strip()):
|
|
278
|
+
raise Refused("bad_params", f"app must be a bundle id or an app's name, not {args['app']!r}")
|
|
279
|
+
if name == "toggle" and args.get("value") is not None and not isinstance(args["value"], bool):
|
|
280
|
+
raise Refused("bad_params", f"value must be true or false, not {args['value']!r}")
|
|
281
|
+
return checked
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class Noted(str):
|
|
285
|
+
"""A handler's result line with a note for the answer's end."""
|
|
286
|
+
note: str = ""
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
class Unchanged(str):
|
|
290
|
+
"""A handler's result that did nothing on the phone (`toggle` to the state it's already in): it is
|
|
291
|
+
the whole answer, with no change block."""
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class Toggled(Noted):
|
|
295
|
+
"""`toggle`'s result, `Tapped <line>.` until the tree is read: then the first line says what the switch did."""
|
|
296
|
+
ref: str = ""
|
|
297
|
+
before: ui.Element = ui.Element("", "switch") # the switch as it was tapped
|
|
298
|
+
want: bool | None = None # the `value` asked for
|
|
299
|
+
flipped: bool | None = None # where a tap should have taken it
|
|
300
|
+
|
|
301
|
+
def verdict(self, now: ui.Element, alert: bool) -> str:
|
|
302
|
+
"""The first line from the switch as it reads after the tap. Raises when a wanted value wasn't reached
|
|
303
|
+
(unless an alert is up: it may be asking to confirm)."""
|
|
304
|
+
line = now.line(actions=False)
|
|
305
|
+
if now.checked == self.flipped:
|
|
306
|
+
return f"Toggled {line}."
|
|
307
|
+
if self.want is not None and not alert:
|
|
308
|
+
raise Refused("not_actionable", f"no_action: {line} didn't change")
|
|
309
|
+
return f"Tapped {line}, but it is still checked={'true' if now.checked else 'false'}."
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
class Typed(Noted):
|
|
313
|
+
"""`type_text`'s result, `Typed into <line>.` until the tree is read: then the first line also gives the value."""
|
|
314
|
+
field: ui.Element = ui.Element("", "input") # the input as it was when typed into
|
|
315
|
+
|
|
316
|
+
def same_field(self, now: ui.Element) -> bool:
|
|
317
|
+
"""Is `now` the field typed into (the ref may have gone to another input since)? The same ref and
|
|
318
|
+
role, and the same identifier, or without one the same label."""
|
|
319
|
+
old = self.field
|
|
320
|
+
return (now.ref, now.role, now.identifier) == (old.ref, old.role, old.identifier) \
|
|
321
|
+
and (bool(old.identifier) or now.label == old.label)
|
|
322
|
+
|
|
323
|
+
def verdict(self, now: ui.Element) -> str:
|
|
324
|
+
line = now.line(actions=False)
|
|
325
|
+
if now.secure:
|
|
326
|
+
return f"Typed into {line}. The value is hidden (secure field)."
|
|
327
|
+
return f"Typed into {line}. Value is now {json.dumps(now.value, ensure_ascii=False)}."
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
class Scrolled(Noted):
|
|
331
|
+
"""`scroll`'s result, with a note for the answer when it stopped early."""
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
# What `open_app "Settings"` means: the built-in apps aren't in the installed list.
|
|
335
|
+
SYSTEM_APPS = {"Settings": "com.apple.Preferences", "Safari": "com.apple.mobilesafari",
|
|
336
|
+
"Calculator": "com.apple.calculator", "Camera": "com.apple.camera",
|
|
337
|
+
"Photos": "com.apple.mobileslideshow", "Messages": "com.apple.MobileSMS",
|
|
338
|
+
"Mail": "com.apple.mobilemail", "Maps": "com.apple.Maps", "App Store": "com.apple.AppStore",
|
|
339
|
+
"Clock": "com.apple.mobiletimer", "Notes": "com.apple.mobilenotes"}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def app_line(a: dict) -> str:
|
|
343
|
+
return f"{a['bundle_id']} {a['name']} {a['version']} ({a['build']})"
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _target(a: dict) -> ui.Target:
|
|
347
|
+
return a["target"] # parsed by _checked; tap and long_press require it
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
# Agent actions: tool -> (what it does on the session, seconds to settle before the look; None: no look)
|
|
351
|
+
ACTIONS = {
|
|
352
|
+
"open_app": (None, LONG_CAP), # None: the handler is a Daemon method (Daemon._handlers), which needs the agent's list
|
|
353
|
+
"close_app": (None, None),
|
|
354
|
+
"toggle": (None, CAP),
|
|
355
|
+
"back": (None, CAP),
|
|
356
|
+
"tap": (None, CAP),
|
|
357
|
+
"long_press": (None, CAP),
|
|
358
|
+
"type_text": (None, CAP),
|
|
359
|
+
"scroll": (None, CAP),
|
|
360
|
+
"swipe": (lambda s, a: s.swipe(float(a["x0"]), float(a["y0"]), float(a["x1"]), float(a["y1"]),
|
|
361
|
+
int(float(a.get("seconds", 0.35)) * 1000)), CAP),
|
|
362
|
+
"press_button": (lambda s, a: s.press_button(a["name"]), CAP),
|
|
363
|
+
"alert": (lambda s, a: s.alert(a["action"]), CAP),
|
|
364
|
+
"open_url": (lambda s, a: s.open_url(a["url"]), LONG_CAP),
|
|
365
|
+
"rotate": (lambda s, a: s.rotate(a["to"]), CAP),
|
|
366
|
+
}
|
|
367
|
+
SPRINGBOARD = "com.apple.springboard"
|
|
368
|
+
PROMPT_FRONT = "(a given app's prompt)" # Daemon._front: a given app's own system prompt is in front
|
|
369
|
+
PROMPT_RETURN_WAIT = 2.0 # seconds the app may take to come back once its prompt is answered
|
|
370
|
+
FRONT_RETRIES, FRONT_RETRY = 16, 0.5 # see Daemon._front: as long as a notification banner stays
|
|
371
|
+
MIN_VISIBLE = 10.0 # points: less of an element showing than this (high or wide) is no place to touch
|
|
372
|
+
# The status bar across the top of a phone held upright: a touch there scrolls the app's list to its top (iOS),
|
|
373
|
+
# whatever row lies under it. It is in no app's tree, so a row half under it looked reachable: a tap on Settings'
|
|
374
|
+
# "Notifications" row at y 10 scrolled the list instead (seen on the phone 2026-11-11). 50 pt covers the status
|
|
375
|
+
# bar of the current iPhones (47 to 54); in landscape an iPhone shows none.
|
|
376
|
+
STATUS_BAR = 50.0
|
|
377
|
+
MOVES_CONTENT = ("scroll", "swipe") # a drag: identical elements can't be told apart after these
|
|
378
|
+
# These may show the same screen again, somewhere else in it, or another: the elements say which. Calling each a
|
|
379
|
+
# move gave every element without an identifier a new ref, as a `+` and a `-` line for the same thing (seen on
|
|
380
|
+
# the phone 2026-11-10: open_url to the screen already open).
|
|
381
|
+
MAY_MOVE_CONTENT = ("back", "open_app", "open_url", "press_button", "rotate")
|
|
382
|
+
READS = ("look", "apps")
|
|
383
|
+
BUNDLE_ID = re.compile(r"^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+$")
|
|
384
|
+
SPECIAL = ("wait", "wait_for") # sleep before they answer, so they don't hold the phone's queue meanwhile
|
|
385
|
+
REQUIRED = {"rotate": ("to",), "open_app": ("app",), "close_app": ("app",), "toggle": ("target",), "wait": ("seconds",),
|
|
386
|
+
"type_text": ("text",), "tap": ("target",), "long_press": ("target",),
|
|
387
|
+
"swipe": ("x0", "y0", "x1", "y1"), "press_button": ("name",), "alert": ("action",),
|
|
388
|
+
"open_url": ("url",)}
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
async def _first(timeout: float, *events: asyncio.Event) -> None:
|
|
392
|
+
"""Wait until any of the events is set, or `timeout` seconds."""
|
|
393
|
+
waits = [asyncio.create_task(e.wait()) for e in events]
|
|
394
|
+
try:
|
|
395
|
+
await asyncio.wait(waits, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
|
|
396
|
+
finally:
|
|
397
|
+
for w in waits:
|
|
398
|
+
w.cancel()
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
class Daemon:
|
|
402
|
+
def __init__(self, session, control: Controller | None = None, log=print, heartbeat: float = 5.0,
|
|
403
|
+
backoff: tuple = (1.0, 2.0, 5.0, 10.0), ready_wait: float = 60.0,
|
|
404
|
+
stream_linger: float = 30.0, stream_retry: float = 1.0, settle_still: float = STILL,
|
|
405
|
+
settle_first_frame: float = 1.0, settle_fallback: float = 0.6,
|
|
406
|
+
settle_quiet: float = MIN_QUIET, shot_wait: float = 1.0, shot_fresh: float = 1.0,
|
|
407
|
+
pace_tick: float | None = PACE_TICK, pace_quiet: tuple[float, float, float] = PACE_QUIET) -> None:
|
|
408
|
+
self.session = session
|
|
409
|
+
# the stream's pace (see STILL_FPS): how often it's checked (None: never), and the frame rate the phone
|
|
410
|
+
# streams at now (None until connected)
|
|
411
|
+
self.pace_tick, self.pace_quiet = pace_tick, pace_quiet
|
|
412
|
+
self._fps: int | None = None
|
|
413
|
+
self._person_at = self._changed_at = -math.inf # monotonic: a person's last input, the screen's last change
|
|
414
|
+
self._pace_seen: Frame | None = None # the newest frame at the last check
|
|
415
|
+
# screenshots: how long to wait for a stream frame before asking the phone, and how old one may be
|
|
416
|
+
self.shot_wait, self.shot_fresh = shot_wait, shot_fresh
|
|
417
|
+
self.unsettled = False # the last action returned without a settle: the screen may still be moving, for every client
|
|
418
|
+
# settle: how long a still screen is quiet, how long to wait for a first frame, and the
|
|
419
|
+
# blind sleep used when the stream gives no frames at all
|
|
420
|
+
self.settle_still, self.settle_first_frame, self.settle_fallback = settle_still, settle_first_frame, settle_fallback
|
|
421
|
+
self.settle_quiet = settle_quiet
|
|
422
|
+
self.control = control or Controller()
|
|
423
|
+
self.hub = FrameHub()
|
|
424
|
+
self.pump = StreamPump(self._open_stream, self.hub, linger=stream_linger, retry=stream_retry, log=log,
|
|
425
|
+
suspended=self._fed_by_video)
|
|
426
|
+
self.log = log
|
|
427
|
+
# A trial switch (TAPWRIGHT_AGENT_STREAM=0): tool calls don't open the JPEG stream, so an action settles
|
|
428
|
+
# on a short wait and a tree read. To learn whether the stream is what stalls requests over Wi-Fi.
|
|
429
|
+
self.agent_stream = os.environ.get("TAPWRIGHT_AGENT_STREAM", "1") != "0"
|
|
430
|
+
self._agent_touches = AgentTouches()
|
|
431
|
+
self._turned: tuple | None = None # ((frame seq, degrees), the turned JPEG): see turned()
|
|
432
|
+
self.heartbeat, self.backoff, self.ready_wait = heartbeat, backoff, ready_wait
|
|
433
|
+
self.state, self.detail = "connecting", None
|
|
434
|
+
self.preset = os.environ.get("TAPWRIGHT_PRESET", "balanced") # (a trial switch: the stream's preset at start)
|
|
435
|
+
if self.preset not in PRESETS:
|
|
436
|
+
self.preset = "balanced"
|
|
437
|
+
self.ws_clients = 0
|
|
438
|
+
self.video = None # a tapwright.video.VideoHub while the H.264 live view is set up (see start_video)
|
|
439
|
+
self.want_video = False # `tapwright serve --video`: set the video up once the phone is connected
|
|
440
|
+
# `--settle-on-video`: actions watch the video's frames and the JPEG stream stays closed. Off unless asked:
|
|
441
|
+
# a mirror that stalls sends nothing, exactly as a still screen does, so an action could answer "No
|
|
442
|
+
# visible change" for a screen that changed. Not for use until a check against the phone guards that.
|
|
443
|
+
self.settle_on_video = False
|
|
444
|
+
self._receiver = None # the AirPlay receiver process that feeds `video`
|
|
445
|
+
self._video_task: asyncio.Task | None = None
|
|
446
|
+
self._feeder: PictureFeeder | None = None # feeds `hub` from `video` while the hub is live (_start_feeder)
|
|
447
|
+
self._feeder_task: asyncio.Task | None = None
|
|
448
|
+
self.last_request = time.monotonic()
|
|
449
|
+
self._ever_connected = False
|
|
450
|
+
# why the last connect couldn't finish, when the phone said: a PhoneLocked, or an AppNotAnswering
|
|
451
|
+
self.stuck: PhoneLocked | AppNotAnswering | None = None
|
|
452
|
+
self._connected = asyncio.Event()
|
|
453
|
+
self._lost = asyncio.Event()
|
|
454
|
+
self._wake = asyncio.Event() # connect again now, without waiting out the backoff
|
|
455
|
+
self._connecting = False # a connect is under way
|
|
456
|
+
self._attempts = 0 # connects that ended, either way
|
|
457
|
+
self._scopes: dict[tuple[str, ...], dict[str, str]] = {} # each app list seen, as {bundle id: shown}
|
|
458
|
+
# {agent: the app its last touch near the top was meant for}, for a look: false answer that asked once
|
|
459
|
+
# {agent: (the app, when, the person generation then)}: see _recheck_later and _check_again
|
|
460
|
+
self._recheck: dict[str, tuple[tuple[str, str], float, int]] = {}
|
|
461
|
+
self._taken_notes: dict[str, str] = {} # {agent: a note _check_again found for its look}
|
|
462
|
+
self._queues: set = set()
|
|
463
|
+
self._handlers = {"open_app": self._open_app, "close_app": self._close_app, "toggle": self._toggle,
|
|
464
|
+
"tap": self._tap, "long_press": self._long_press, "back": self._back, "type_text": self._type_text, "scroll": self._scroll}
|
|
465
|
+
|
|
466
|
+
# ---- status and events ----
|
|
467
|
+
def status(self) -> dict:
|
|
468
|
+
w, h = self.session.screen
|
|
469
|
+
via = (getattr(self.session.device, "via", None) or "").lower()
|
|
470
|
+
via = ("usb" if via == "usb" else "wifi" if via == "local network"
|
|
471
|
+
else "tailscale" if via.startswith("network") else None)
|
|
472
|
+
return {"type": "status", "protocol": PROTOCOL, "tool": "Tapwright", "state": self.state, "detail": self.detail, "via": via,
|
|
473
|
+
"fps": self.hub.fps(), "driver": self.control.driver, "paused": self.control.paused,
|
|
474
|
+
"size": {"w": w, "h": h}, "quality": self.preset,
|
|
475
|
+
"accepts": [*ACCEPTS, "video"] if self.video is not None and self.video.live else ACCEPTS}
|
|
476
|
+
|
|
477
|
+
def subscribe(self) -> asyncio.Queue:
|
|
478
|
+
self._asked_for()
|
|
479
|
+
queue: asyncio.Queue = asyncio.Queue(maxsize=200)
|
|
480
|
+
self._queues.add(queue)
|
|
481
|
+
return queue
|
|
482
|
+
|
|
483
|
+
def unsubscribe(self, queue: asyncio.Queue) -> None:
|
|
484
|
+
self._queues.discard(queue)
|
|
485
|
+
|
|
486
|
+
def broadcast(self, message: dict) -> None:
|
|
487
|
+
"""To every live view; one that can't keep up loses its oldest messages."""
|
|
488
|
+
for queue in list(self._queues):
|
|
489
|
+
if queue.full():
|
|
490
|
+
queue.get_nowait()
|
|
491
|
+
queue.put_nowait(message)
|
|
492
|
+
|
|
493
|
+
def _set_state(self, state: str, detail: str | None = None) -> None:
|
|
494
|
+
changed = (state, detail) != (self.state, self.detail)
|
|
495
|
+
self.state, self.detail = state, detail
|
|
496
|
+
if state in ("live", "locked"):
|
|
497
|
+
self._connected.set()
|
|
498
|
+
else:
|
|
499
|
+
self._connected.clear()
|
|
500
|
+
if changed:
|
|
501
|
+
self.broadcast(self.status())
|
|
502
|
+
|
|
503
|
+
def _point(self, point) -> dict | None:
|
|
504
|
+
if not point:
|
|
505
|
+
return None
|
|
506
|
+
w, h = self.session.screen
|
|
507
|
+
return {"x": round(point[0] / w, 4), "y": round(point[1] / h, 4)}
|
|
508
|
+
|
|
509
|
+
def _announce(self, by: str, kind: str, label: str | None = None, point=None, ok: bool = True,
|
|
510
|
+
code: str | None = None, caption: str | None = None, reason: str | None = None) -> None:
|
|
511
|
+
message = {"type": "action", "by": by, "kind": kind, "at": int(time.time() * 1000), "ok": ok}
|
|
512
|
+
if label:
|
|
513
|
+
message["label"] = label
|
|
514
|
+
if caption:
|
|
515
|
+
message["caption"] = caption
|
|
516
|
+
if reason:
|
|
517
|
+
message["reason"] = reason
|
|
518
|
+
if point:
|
|
519
|
+
message["point"] = point
|
|
520
|
+
if code:
|
|
521
|
+
message["code"] = code
|
|
522
|
+
self.broadcast(message)
|
|
523
|
+
|
|
524
|
+
# ---- the connection ----
|
|
525
|
+
def lost(self, reason: "str | Exception") -> None:
|
|
526
|
+
"""A request found the line dead: the supervisor reconnects. When the request was cut because the app in
|
|
527
|
+
front doesn't answer (AppNotAnswering), that is what agents and the live view are told meanwhile."""
|
|
528
|
+
stuck = reason if isinstance(reason, AppNotAnswering) else None
|
|
529
|
+
if stuck is not None:
|
|
530
|
+
self.stuck = stuck
|
|
531
|
+
if self.state in ("live", "locked"):
|
|
532
|
+
self.log(f"connection lost: {reason}")
|
|
533
|
+
self._set_state("not-answering" if stuck is not None else "reconnecting", str(reason)[:200])
|
|
534
|
+
for view in self.control.clients.values():
|
|
535
|
+
view.dropped = True # every agent's list is from before the drop
|
|
536
|
+
self._lost.set()
|
|
537
|
+
|
|
538
|
+
def _not_connected(self) -> Refused:
|
|
539
|
+
"""No phone found or not set up: the agent can't fix that by waiting. Otherwise the link
|
|
540
|
+
is coming back."""
|
|
541
|
+
if self.state == "no-device":
|
|
542
|
+
return Refused("not_connected", f"no phone to drive ({self.detail}); fix that, then try again")
|
|
543
|
+
if isinstance(self.stuck, PasscodeNeeded):
|
|
544
|
+
return Refused("locked", "the phone is asking for its passcode to allow UI automation; ask the user "
|
|
545
|
+
"to enter it on the phone")
|
|
546
|
+
if isinstance(self.stuck, PhoneLocked):
|
|
547
|
+
return Refused("locked", "the phone is locked; ask the user to unlock it")
|
|
548
|
+
if isinstance(self.stuck, AppNotAnswering):
|
|
549
|
+
return Refused("not_answering", f"{self.stuck}. close_app can still close it; then open_app it again.")
|
|
550
|
+
return Refused("reconnecting", f"not connected to the phone ({self.detail or self.state}); "
|
|
551
|
+
"wait a few seconds, then look")
|
|
552
|
+
|
|
553
|
+
def _require_connected(self) -> None:
|
|
554
|
+
if not self._connected.is_set():
|
|
555
|
+
raise self._not_connected()
|
|
556
|
+
|
|
557
|
+
async def _connected_only(self, fn):
|
|
558
|
+
"""Run `fn` only if the line is up when its turn comes: a reconnect holds the queue,
|
|
559
|
+
and whatever waited for it must not reconnect on its own."""
|
|
560
|
+
self._require_connected()
|
|
561
|
+
return await fn()
|
|
562
|
+
|
|
563
|
+
def _asked_for(self) -> bool:
|
|
564
|
+
"""A tool call or a viewer wants the phone while it waits behind the passcode screen: put that screen up
|
|
565
|
+
now if it has gone (the starts thin out when nobody asks, see device.PROMPT_PAUSES). True when a start
|
|
566
|
+
was asked for."""
|
|
567
|
+
if isinstance(self.stuck, PasscodeNeeded):
|
|
568
|
+
hurry = getattr(self.session.device, "hurry", None)
|
|
569
|
+
if hurry is not None:
|
|
570
|
+
hurry()
|
|
571
|
+
self._wake.set()
|
|
572
|
+
return True
|
|
573
|
+
return False
|
|
574
|
+
|
|
575
|
+
async def _next_start(self) -> None:
|
|
576
|
+
"""Wait for a connect that began after now to end (or for the line to come up). What the last start
|
|
577
|
+
found may be minutes old: the person may have typed the passcode since, with nothing started to notice
|
|
578
|
+
(seen on the phone 2026-11-10: a tap answered `locked` 20 s before the phone was back). A connect
|
|
579
|
+
already under way may have looked before it was asked to hurry, so the one after it counts."""
|
|
580
|
+
need = self._attempts + (2 if self._connecting else 1)
|
|
581
|
+
deadline = time.monotonic() + self.ready_wait
|
|
582
|
+
while self._attempts < need and not self._connected.is_set() and time.monotonic() < deadline:
|
|
583
|
+
await asyncio.sleep(0.05)
|
|
584
|
+
|
|
585
|
+
async def wait_connected(self) -> None:
|
|
586
|
+
if self._asked_for():
|
|
587
|
+
await self._next_start()
|
|
588
|
+
if self.state == "no-device" or self.stuck is not None: # waiting won't fix either
|
|
589
|
+
raise self._not_connected()
|
|
590
|
+
try:
|
|
591
|
+
await asyncio.wait_for(self._connected.wait(), self.ready_wait)
|
|
592
|
+
except asyncio.TimeoutError:
|
|
593
|
+
raise self._not_connected() from None
|
|
594
|
+
|
|
595
|
+
async def _connect(self) -> None:
|
|
596
|
+
"""A working WebDriverAgent session: a new WebDriverAgent if the tunnel still works,
|
|
597
|
+
otherwise everything from scratch."""
|
|
598
|
+
async with self.control.exclusive():
|
|
599
|
+
s = self.session
|
|
600
|
+
if s.device.rsd is not None:
|
|
601
|
+
s.sid = None
|
|
602
|
+
try:
|
|
603
|
+
await s.ensure()
|
|
604
|
+
except (NotSetUp, PhoneLocked, AppNotAnswering): # the tunnel is fine: rebuilding it won't help
|
|
605
|
+
raise
|
|
606
|
+
except Exception as e:
|
|
607
|
+
self.log(f"restarting WebDriverAgent didn't work ({e}); reconnecting")
|
|
608
|
+
await s.reset()
|
|
609
|
+
await s.ensure()
|
|
610
|
+
else:
|
|
611
|
+
await s.ensure()
|
|
612
|
+
await s.settings({**PRESETS[self.preset], **NO_IDLE_WAIT})
|
|
613
|
+
self._fps = self._preset_fps()
|
|
614
|
+
await s.read_scale(refresh=True) # once per connection, so every header says @3x, not only SpringBoard's
|
|
615
|
+
await s.read_app_names(refresh=True) # so the header's App: line says "Settings", not a bundle id
|
|
616
|
+
|
|
617
|
+
async def supervise(self, stop: asyncio.Event) -> None:
|
|
618
|
+
"""Connect, then keep checking; reconnect whenever the line drops. Runs until `stop`."""
|
|
619
|
+
self._agent_touches.start() # find Tapline's hosts, if any, before the first touch needs them
|
|
620
|
+
pacer = asyncio.create_task(self._pace(stop), name="pace") if self.pace_tick else None
|
|
621
|
+
watch = asyncio.create_task(self._watch_loop(stop), name="loop-watch") if self.pace_tick else None
|
|
622
|
+
try:
|
|
623
|
+
await self._supervise(stop)
|
|
624
|
+
finally:
|
|
625
|
+
for helper in (pacer, watch):
|
|
626
|
+
if helper is not None:
|
|
627
|
+
helper.cancel()
|
|
628
|
+
await asyncio.gather(helper, return_exceptions=True)
|
|
629
|
+
|
|
630
|
+
async def _watch_loop(self, stop: asyncio.Event) -> None:
|
|
631
|
+
"""Say in the log when this process's event loop was held up. The tunnel to the phone is a TCP/IP stack
|
|
632
|
+
in pure Python on this same loop: while something holds the loop, no packet moves, and a held loop looks
|
|
633
|
+
from outside like a phone that doesn't answer."""
|
|
634
|
+
loop = asyncio.get_running_loop()
|
|
635
|
+
worst, told = 0.0, loop.time()
|
|
636
|
+
while not stop.is_set():
|
|
637
|
+
before = loop.time()
|
|
638
|
+
await asyncio.sleep(LOOP_TICK)
|
|
639
|
+
worst = max(worst, loop.time() - before - LOOP_TICK)
|
|
640
|
+
if worst >= LOOP_HELD and loop.time() - told >= LOOP_SAY_EVERY:
|
|
641
|
+
self.log(f"the event loop was held for {worst:.2f} s; the tunnel's packets waited meanwhile")
|
|
642
|
+
worst, told = 0.0, loop.time()
|
|
643
|
+
|
|
644
|
+
# ---- the stream's pace ----
|
|
645
|
+
def _preset_fps(self) -> int:
|
|
646
|
+
return int(PRESETS[self.preset]["mjpegServerFramerate"])
|
|
647
|
+
|
|
648
|
+
async def _set_fps(self, fps: int) -> None:
|
|
649
|
+
await self.session.settings({"mjpegServerFramerate": fps})
|
|
650
|
+
self._fps = fps
|
|
651
|
+
|
|
652
|
+
async def _quicken(self) -> None:
|
|
653
|
+
"""Back to the preset's rate before an action, so its settle (and the person's picture) has every frame.
|
|
654
|
+
Only after a quiet spell; a failure to change the rate never stops the action."""
|
|
655
|
+
if self._fps is not None and self._fps < self._preset_fps() and not self._fed_by_video():
|
|
656
|
+
try:
|
|
657
|
+
await self._set_fps(self._preset_fps())
|
|
658
|
+
except ConnectionLost:
|
|
659
|
+
raise
|
|
660
|
+
except DeviceError as e:
|
|
661
|
+
self.log(f"pace: couldn't raise the frame rate: {e}")
|
|
662
|
+
|
|
663
|
+
def _fed_by_video(self) -> bool:
|
|
664
|
+
"""Is the AirPlay video the hub's source now (so the MJPEG stream stays closed)?"""
|
|
665
|
+
return self._feeder is not None and self._feeder.feeding
|
|
666
|
+
|
|
667
|
+
async def _pace(self, stop: asyncio.Event) -> None:
|
|
668
|
+
while not stop.is_set():
|
|
669
|
+
await _first(self.pace_tick, stop)
|
|
670
|
+
if stop.is_set():
|
|
671
|
+
return
|
|
672
|
+
try:
|
|
673
|
+
await self._pace_once()
|
|
674
|
+
except ConnectionLost as e:
|
|
675
|
+
self.lost(e)
|
|
676
|
+
except Refused:
|
|
677
|
+
pass # not connected, or paused: the next check tries again
|
|
678
|
+
except Exception as e:
|
|
679
|
+
self.log(f"pace: {e}")
|
|
680
|
+
|
|
681
|
+
async def _pace_once(self) -> None:
|
|
682
|
+
"""Slow the stream when nothing has happened for a while, and quicken it when the screen changes."""
|
|
683
|
+
if self._fed_by_video():
|
|
684
|
+
return # the MJPEG stream is closed: there is no frame rate to set
|
|
685
|
+
now = time.monotonic()
|
|
686
|
+
frame = self.hub.latest
|
|
687
|
+
if frame is not None and frame is not self._pace_seen:
|
|
688
|
+
if not alike(frame, self._pace_seen): # a first frame counts as a change: nothing says it's still yet
|
|
689
|
+
self._changed_at = now
|
|
690
|
+
self._pace_seen = frame
|
|
691
|
+
if self.state != "live" or self._fps is None or frame is None or self.hub.fps() == 0:
|
|
692
|
+
return # nothing streams, so the pace costs nothing
|
|
693
|
+
agent, person, still = self.pace_quiet
|
|
694
|
+
busy = (now - self.last_request < agent or now - self._person_at < person or now - self._changed_at < still)
|
|
695
|
+
want = self._preset_fps() if busy else min(STILL_FPS, self._preset_fps())
|
|
696
|
+
if want != self._fps:
|
|
697
|
+
await self.control.read(lambda: self._connected_only(lambda: self._set_fps(want)))
|
|
698
|
+
|
|
699
|
+
async def _supervise(self, stop: asyncio.Event) -> None:
|
|
700
|
+
attempt = failures = 0
|
|
701
|
+
while not stop.is_set():
|
|
702
|
+
if self.state not in ("live", "locked"):
|
|
703
|
+
self._connecting = True
|
|
704
|
+
try:
|
|
705
|
+
try:
|
|
706
|
+
await self._connect()
|
|
707
|
+
finally:
|
|
708
|
+
self._connecting = False
|
|
709
|
+
self._attempts += 1
|
|
710
|
+
except NotSetUp as e:
|
|
711
|
+
self._set_state("no-device", f"needs setup: {e}")
|
|
712
|
+
except PhoneNotFound:
|
|
713
|
+
self._set_state("no-device", "phone not found: plug it in and unlock it")
|
|
714
|
+
except (PhoneLocked, AppNotAnswering) as e:
|
|
715
|
+
if self.stuck is None:
|
|
716
|
+
self.log(f"connected, but {e}")
|
|
717
|
+
self.stuck = e
|
|
718
|
+
state = "reconnecting" if self._ever_connected else "connecting"
|
|
719
|
+
self._set_state("not-answering" if isinstance(e, AppNotAnswering) else state, str(e)[:200])
|
|
720
|
+
except Exception as e:
|
|
721
|
+
self.stuck = None
|
|
722
|
+
self._set_state("reconnecting" if self._ever_connected else "connecting", str(e)[:200])
|
|
723
|
+
else:
|
|
724
|
+
attempt = failures = 0
|
|
725
|
+
self.stuck = None
|
|
726
|
+
self._ever_connected = True
|
|
727
|
+
self._lost.clear() # a drop noticed while reconnecting is already dealt with
|
|
728
|
+
self._set_state("live")
|
|
729
|
+
self.log(f"connected over {self.session.device.via}")
|
|
730
|
+
if self.want_video and self._video_task is None:
|
|
731
|
+
self._video_task = asyncio.create_task(self.start_video(), name="video")
|
|
732
|
+
continue
|
|
733
|
+
await _first(self.backoff[min(attempt, len(self.backoff) - 1)], stop, self._wake)
|
|
734
|
+
self._wake.clear()
|
|
735
|
+
attempt += 1
|
|
736
|
+
continue
|
|
737
|
+
await _first(self.heartbeat, stop, self._lost)
|
|
738
|
+
if stop.is_set():
|
|
739
|
+
break
|
|
740
|
+
if self._lost.is_set():
|
|
741
|
+
self._lost.clear()
|
|
742
|
+
if self.state in ("live", "locked"):
|
|
743
|
+
self._set_state("reconnecting", "connection lost")
|
|
744
|
+
continue
|
|
745
|
+
try:
|
|
746
|
+
locked = await self.control.read(lambda: self._connected_only(self.session.locked))
|
|
747
|
+
except ConnectionLost as e:
|
|
748
|
+
self.lost(e)
|
|
749
|
+
except Refused:
|
|
750
|
+
pass
|
|
751
|
+
except Exception as e:
|
|
752
|
+
failures += 1
|
|
753
|
+
self.log(f"heartbeat: {e}")
|
|
754
|
+
if failures >= 3:
|
|
755
|
+
failures = 0
|
|
756
|
+
self.lost(f"WebDriverAgent keeps failing: {e}")
|
|
757
|
+
else:
|
|
758
|
+
failures = 0
|
|
759
|
+
if not self._lost.is_set() and self.state in ("live", "locked"):
|
|
760
|
+
if locked:
|
|
761
|
+
self._set_state("locked", "unlock the phone")
|
|
762
|
+
else:
|
|
763
|
+
self._set_state("live")
|
|
764
|
+
|
|
765
|
+
# ---- agent tools ----
|
|
766
|
+
async def run_tool(self, client_id: str, name: str, args: dict, scope: "Scope | None" = None) -> dict:
|
|
767
|
+
"""One agent tool call: {"text": str, "image": base64 JPEG or None, "image_size": [w, h] or None}.
|
|
768
|
+
Raises Refused with a code from errors.CODES; any other failure is `internal` (logged).
|
|
769
|
+
`scope` is the apps this agent was given (personal mode); None is a dedicated device: no limit."""
|
|
770
|
+
self.last_request = time.monotonic()
|
|
771
|
+
try:
|
|
772
|
+
# The stream is held open for the length of every tool call; the pump's own linger keeps
|
|
773
|
+
# it open for 30 s after the last one, so the settle always has frames to read.
|
|
774
|
+
# Not when there is no phone: the pump would only retry opening it.
|
|
775
|
+
async with self.pump.watching() if self._connected.is_set() and self.agent_stream else nullcontext():
|
|
776
|
+
await self._check_again(client_id, name) # before personal mode's front check: see there
|
|
777
|
+
if scope is None:
|
|
778
|
+
return await self._run_tool(client_id, name, args)
|
|
779
|
+
return await self._within(scope, client_id, name, args)
|
|
780
|
+
except Refused:
|
|
781
|
+
raise
|
|
782
|
+
except DeviceError as e: # WebDriverAgent or the link answered with an error; not our bug
|
|
783
|
+
self.log(f"{name}: the phone side failed: {e}")
|
|
784
|
+
raise Refused("internal", f"the phone side failed: {e}") from e
|
|
785
|
+
except Exception as e: # one failed step never takes the daemon down
|
|
786
|
+
self.log(f"{name} failed unexpectedly:\n{traceback.format_exc()}")
|
|
787
|
+
raise Refused("internal", f"{name} failed unexpectedly: {e}. This is a bug; report it.") from e
|
|
788
|
+
|
|
789
|
+
# ---- personal mode: the agent stays in the apps it was given ----
|
|
790
|
+
async def _given(self, scope: "Scope") -> dict[str, str]:
|
|
791
|
+
"""{bundle id: how to show it} of the apps in a scope. An entry that looks like a bundle id is taken as
|
|
792
|
+
it is; a name is looked up among the installed apps, once."""
|
|
793
|
+
if scope.apps not in self._scopes:
|
|
794
|
+
given: dict[str, str] = {}
|
|
795
|
+
for entry in scope.apps:
|
|
796
|
+
if BUNDLE_ID.match(entry):
|
|
797
|
+
given[entry] = entry
|
|
798
|
+
continue
|
|
799
|
+
await self.wait_connected()
|
|
800
|
+
try:
|
|
801
|
+
bundle, shown = await self._app(entry, partial=False)
|
|
802
|
+
except Refused as e:
|
|
803
|
+
raise Refused("unsupported", f'the app list this agent was started with names "{entry}", which '
|
|
804
|
+
f"can't be used: {e}") from None
|
|
805
|
+
given[bundle] = shown
|
|
806
|
+
self._scopes[scope.apps] = given
|
|
807
|
+
return self._scopes[scope.apps]
|
|
808
|
+
|
|
809
|
+
async def _front(self, given: dict[str, str], prompt=None) -> str | None:
|
|
810
|
+
"""The app in front, for personal mode's question before a touch. WebDriverAgent answers SpringBoard
|
|
811
|
+
while a notification banner slides over the app in front (seen on the phone 2026-11-10: twice in forty
|
|
812
|
+
calls, each a false `outside_apps`; a banner stays for some seconds). When it says SpringBoard and the phone's
|
|
813
|
+
own process list has one of the given apps in the foreground, it is asked again, FRONT_RETRIES times
|
|
814
|
+
FRONT_RETRY s apart at most. Only WebDriverAgent's answer lets a call through; the process list never
|
|
815
|
+
does, and on the real home screen it has no given app, so that refusal isn't delayed. `prompt`: asked first
|
|
816
|
+
whether the screen is a given app's own permission prompt (see _prompt_of_given): then PROMPT_FRONT at once,
|
|
817
|
+
with no wait, since a prompt doesn't leave by itself as a banner does."""
|
|
818
|
+
front = await self._read(self.session.foreground_bundle_id)
|
|
819
|
+
if front != SPRINGBOARD:
|
|
820
|
+
return front
|
|
821
|
+
try:
|
|
822
|
+
running = set(await asyncio.wait_for(self.session.device.foreground_apps(), 5))
|
|
823
|
+
except Exception: # no process list in time: WebDriverAgent's word stands
|
|
824
|
+
return front
|
|
825
|
+
if running & set(given) and prompt is not None and await prompt():
|
|
826
|
+
return PROMPT_FRONT
|
|
827
|
+
for _ in range(FRONT_RETRIES if running & set(given) else 0):
|
|
828
|
+
await asyncio.sleep(FRONT_RETRY)
|
|
829
|
+
front = await self._read(self.session.foreground_bundle_id)
|
|
830
|
+
if front != SPRINGBOARD:
|
|
831
|
+
break
|
|
832
|
+
return front
|
|
833
|
+
|
|
834
|
+
async def _prompt_of_given(self, given: dict[str, str], view, read: bool) -> bool:
|
|
835
|
+
"""Is the system alert on screen a given app's own: a permission prompt the app raised ("“Expo Go” Would
|
|
836
|
+
Like to Send You Notifications")? iOS draws it in SpringBoard, so personal mode saw the home screen in front
|
|
837
|
+
and refused the very `alert` the skill says to answer (a first-time agent stuck on it, 2026-11-11). Only
|
|
838
|
+
when the alert's title names a given app and the phone's process list still has that app in the
|
|
839
|
+
foreground under it. `read`: read the screen first (a touch's check); else the answer's own read counts."""
|
|
840
|
+
if read:
|
|
841
|
+
await self._read(lambda: self._silent_read(view))
|
|
842
|
+
if view.alert_now is None:
|
|
843
|
+
return False
|
|
844
|
+
title = view.alert_now[0]
|
|
845
|
+
named = [b for b in given if (self.session.app_names.get(b) or given[b].split(" (")[0]) in title]
|
|
846
|
+
if not named:
|
|
847
|
+
return False
|
|
848
|
+
try:
|
|
849
|
+
running = set(await asyncio.wait_for(self.session.device.foreground_apps(), 5))
|
|
850
|
+
except Exception:
|
|
851
|
+
return False # the phone can't say: refuse, the safe side
|
|
852
|
+
return bool(running & set(named))
|
|
853
|
+
|
|
854
|
+
async def _back_from_prompt(self, client_id: str, given: dict[str, str], answer: dict) -> dict:
|
|
855
|
+
"""After a given app's own permission prompt is answered, the app comes back as the prompt fades. The
|
|
856
|
+
answer's read can come mid-fade and show SpringBoard, which personal mode withholds; a first-time agent then
|
|
857
|
+
reopened the app for nothing (2026-11-11). So wait for the app, up to PROMPT_RETURN_WAIT s, and answer
|
|
858
|
+
with its screen."""
|
|
859
|
+
shown = APP_LINE.search(answer["text"]) # as the check below decides: the App: line, else the last read's app
|
|
860
|
+
app = shown.group(2) if shown else (self.session.app or (None, None))[1]
|
|
861
|
+
if app in given:
|
|
862
|
+
return answer
|
|
863
|
+
loop = asyncio.get_running_loop()
|
|
864
|
+
until = loop.time() + PROMPT_RETURN_WAIT
|
|
865
|
+
while await self._read(self.session.foreground_bundle_id) not in given:
|
|
866
|
+
if loop.time() >= until:
|
|
867
|
+
return answer
|
|
868
|
+
await asyncio.sleep(0.25)
|
|
869
|
+
fresh = await self._run_tool(client_id, "look", {"screenshot": False})
|
|
870
|
+
return {**fresh, "text": answer["text"].splitlines()[0] + "\n" + fresh["text"]}
|
|
871
|
+
|
|
872
|
+
async def _within(self, scope: "Scope", client_id: str, name: str, args: dict) -> dict:
|
|
873
|
+
"""One tool call in personal mode, failing closed. Nothing runs without an app list. Opening or closing an
|
|
874
|
+
app outside the list is refused. Reading or touching is refused while another app is in front (asked of
|
|
875
|
+
the phone before the call), and an answer never shows another app's screen."""
|
|
876
|
+
if not scope.apps:
|
|
877
|
+
raise Refused("not_allowed", NO_APPS)
|
|
878
|
+
given = await self._given(scope)
|
|
879
|
+
names = ", ".join(given.values())
|
|
880
|
+
front = None
|
|
881
|
+
if name in ("open_app", "close_app") and isinstance(args.get("app"), str) and args["app"].strip():
|
|
882
|
+
if name == "close_app" and isinstance(self.stuck, AppNotAnswering):
|
|
883
|
+
bundle, shown = self.stuck.bundle_id, f"{self.stuck.name} ({self.stuck.bundle_id})"
|
|
884
|
+
else:
|
|
885
|
+
await self.wait_connected()
|
|
886
|
+
bundle, shown = await self._app(args["app"], partial=name == "open_app")
|
|
887
|
+
if bundle not in given:
|
|
888
|
+
raise Refused("not_allowed", f"outside_apps: {shown} isn't one of the apps you were given ({names}).")
|
|
889
|
+
elif name in ACTIONS and name not in SCOPE_FREE or name == "wait_for":
|
|
890
|
+
# A look needs no question first: its own read says which app is in front, and an answer from
|
|
891
|
+
# another app is withheld below. Touches, and wait_for (it reveals whether a text is there), ask.
|
|
892
|
+
await self.wait_connected()
|
|
893
|
+
view = self.control.client(client_id)
|
|
894
|
+
front = await self._front(given, prompt=lambda: self._prompt_of_given(given, view, read=True))
|
|
895
|
+
if front not in given and front != PROMPT_FRONT:
|
|
896
|
+
raise Refused("not_allowed", _outside(front, self.session.app_names.get(front or ""), names))
|
|
897
|
+
answer = await self._run_tool(client_id, name, args)
|
|
898
|
+
if name == "alert" and front == PROMPT_FRONT:
|
|
899
|
+
answer = await self._back_from_prompt(client_id, given, answer)
|
|
900
|
+
if name == "apps": # only its own apps: the rest of the phone is the owner's business
|
|
901
|
+
lines = answer["text"].splitlines()
|
|
902
|
+
kept = [ln for ln in lines if ln.split(" ", 1)[0] in given or not BUNDLE_ID.match(ln.split(" ", 1)[0])]
|
|
903
|
+
return {**answer, "text": "\n".join(kept)}
|
|
904
|
+
# The answer's own App: line says whose screen it lists; without one, the app at the last read does.
|
|
905
|
+
shown = APP_LINE.search(answer["text"])
|
|
906
|
+
app = (shown.group(1), shown.group(2)) if shown else self.session.app
|
|
907
|
+
if (app is not None and app[1] not in given and ("\n" in answer["text"] or answer.get("image"))
|
|
908
|
+
and not (app[1] == SPRINGBOARD
|
|
909
|
+
and await self._prompt_of_given(given, self.control.client(client_id), read=True))):
|
|
910
|
+
if name == "look": # nothing was done, so there is nothing to report but the refusal
|
|
911
|
+
raise Refused("not_allowed", _outside(app[1], app[0], names))
|
|
912
|
+
first = answer["text"].splitlines()[0]
|
|
913
|
+
first = "" if first.startswith("Screen ") else first + "\n" # a look's first line is the screen itself
|
|
914
|
+
return {"text": f"{first}Note: {_outside(app[1], app[0], names)}", "image": None, "image_size": None}
|
|
915
|
+
return answer
|
|
916
|
+
|
|
917
|
+
async def _run_tool(self, client_id: str, name: str, args: dict) -> dict:
|
|
918
|
+
if name not in ACTIONS and name not in READS and name not in SPECIAL:
|
|
919
|
+
raise Refused("unsupported", f"no tool named {name!r}; the tools are "
|
|
920
|
+
f"{', '.join([*READS, *SPECIAL, *ACTIONS])}")
|
|
921
|
+
missing = [k for k in REQUIRED.get(name, ()) if args.get(k) is None]
|
|
922
|
+
if missing:
|
|
923
|
+
raise Refused("bad_params", f"{name} needs {', '.join(missing)}")
|
|
924
|
+
args = _checked(name, args)
|
|
925
|
+
shot = self._shot_args(args, default=name == "look", custom=name == "look") # bad width: refused here
|
|
926
|
+
# A bad target was refused by _checked, before anything is queued or sent. Any tool whose
|
|
927
|
+
# arguments carry `target` goes through here, so the live-read rule in act() covers it too.
|
|
928
|
+
target = args.get("target")
|
|
929
|
+
view = self.control.client(client_id)
|
|
930
|
+
# an agent that hasn't looked yet has no "last look": its first list is whole anyway
|
|
931
|
+
notes = [NOTE_PERSON] if 0 <= view.seen < self.control.person_generation else []
|
|
932
|
+
if name == "close_app" and isinstance(self.stuck, AppNotAnswering):
|
|
933
|
+
return await self._close_stuck(self.stuck, args["app"], notes)
|
|
934
|
+
await self.wait_connected()
|
|
935
|
+
if (taken := self._taken_notes.pop(client_id, None)) is not None:
|
|
936
|
+
notes = [*notes, taken]
|
|
937
|
+
if name == "apps":
|
|
938
|
+
return await self._apps(notes)
|
|
939
|
+
if name == "wait":
|
|
940
|
+
return await self._wait(view, args, notes, shot)
|
|
941
|
+
if name == "wait_for":
|
|
942
|
+
return await self._wait_for(view, args, notes, shot)
|
|
943
|
+
if name == "look":
|
|
944
|
+
await self._settle_pending()
|
|
945
|
+
return await self._look(view, shot, [], notes)
|
|
946
|
+
if self.state == "locked":
|
|
947
|
+
raise Refused("locked", "the phone is locked; ask the user to unlock it")
|
|
948
|
+
call, settle = ACTIONS[name]
|
|
949
|
+
if session_module.TRACE:
|
|
950
|
+
self.log(f"trace {name} begins")
|
|
951
|
+
by_ref = target is not None and target.kind == "ref"
|
|
952
|
+
# these tools at a point act on the element under it, so they need a current list too
|
|
953
|
+
by_lookup = target is not None and (target.kind in ("id", "label", "text")
|
|
954
|
+
or (name in ("type_text", "toggle", "scroll") and target.kind == "point"))
|
|
955
|
+
|
|
956
|
+
fresh, full, box = [True], [False], {}
|
|
957
|
+
|
|
958
|
+
async def act():
|
|
959
|
+
self._require_connected()
|
|
960
|
+
await self._quicken()
|
|
961
|
+
s = self.session
|
|
962
|
+
# a list read mid-redraw can miss elements while the screen looks unchanged since: a target missing
|
|
963
|
+
# from it is read for again before it is not_found
|
|
964
|
+
live = by_lookup and (not self._current(view) or (target.kind != "point" and self._missing(view.items, target)))
|
|
965
|
+
if not live: # a silent re-read settles for itself
|
|
966
|
+
await self._settle_pending()
|
|
967
|
+
s.items = view.items
|
|
968
|
+
fresh[0] = view.seen == self.control.generation
|
|
969
|
+
# never looked, a person acted, or the line dropped: the agent's list may be anything
|
|
970
|
+
full[0] = not fresh[0] or view.dropped or view.read_at == float("-inf")
|
|
971
|
+
if by_ref and view.dropped:
|
|
972
|
+
raise Refused("stale", "the connection to the phone dropped since your last look; look again")
|
|
973
|
+
if by_ref and not fresh[0]:
|
|
974
|
+
raise Refused("stale", "the screen changed since your last look (someone else used the "
|
|
975
|
+
"phone); look again")
|
|
976
|
+
if live:
|
|
977
|
+
# Resolve on the live tree. The agent's refs follow their elements onto it, so `e3` still
|
|
978
|
+
# means the same element when it moved. The view's freshness doesn't change: a person's
|
|
979
|
+
# action still makes refs stale, and `view.shown` (what the agent was told) stays put.
|
|
980
|
+
if await self._silent_read(view) and self._missing(s.items, target):
|
|
981
|
+
# the app may have been mid-redraw: one more look, once it is still, before saying so
|
|
982
|
+
await self._silent_read(view)
|
|
983
|
+
if by_lookup and target.kind != "point" and self._missing(s.items, target):
|
|
984
|
+
if self._missing(s.items, dataclasses.replace(target, index=None)):
|
|
985
|
+
# an alert hides the app's elements: say so, not "not found". (When only the `index` is
|
|
986
|
+
# out of range the alert's own elements matched, and that miss is the answer.)
|
|
987
|
+
self._refuse_behind_alert(view, force=False)
|
|
988
|
+
self._refuse_behind_sheet(target) # before off_screen: scrolling can't help until the sheet is closed
|
|
989
|
+
if not (name == "scroll" and args.get("to") == "target"): # that scroll is how it comes on screen
|
|
990
|
+
if name in KEYBOARD_TOOLS and view.read_keyboard and self._under_keyboard(target):
|
|
991
|
+
# the keyboard hides it, and a short form can't scroll it up: a person closes the keyboard
|
|
992
|
+
# first (a first-time agent was told off_screen for Wayfare's "Create account", 2026-11-11)
|
|
993
|
+
closed = await self._dismiss_keyboard_now(view)
|
|
994
|
+
if not self._missing(s.items, target):
|
|
995
|
+
box["closed"] = closed
|
|
996
|
+
if "closed" not in box:
|
|
997
|
+
self._refuse_off_screen(target)
|
|
998
|
+
# a person would close the keyboard before the touch
|
|
999
|
+
box["note"] = box.pop("closed", "") or await self._close_keyboard(view, name, target)
|
|
1000
|
+
s.last_point = None
|
|
1001
|
+
box["app"], s.top_touch = s.app, False # the app the touch is meant for (see _taken_away)
|
|
1002
|
+
box["mark"] = self._mark() # where the stream stands as the gesture goes out
|
|
1003
|
+
handler = self._handlers.get(name)
|
|
1004
|
+
return await (handler(view, args) if handler else call(s, args))
|
|
1005
|
+
|
|
1006
|
+
async def act_as_agent():
|
|
1007
|
+
# the touches inside are the agent's: Tapline is told before each one (tapline.py). Set only here, so a
|
|
1008
|
+
# person's input from the live view, queued separately, is never announced as the agent's
|
|
1009
|
+
self.session.touch_hook, self.session.banner_guard = self._agent_touches.notify, True
|
|
1010
|
+
try:
|
|
1011
|
+
return await act()
|
|
1012
|
+
finally:
|
|
1013
|
+
self.session.touch_hook, self.session.banner_guard = None, False
|
|
1014
|
+
|
|
1015
|
+
try:
|
|
1016
|
+
result = await self.control.agent(act_as_agent)
|
|
1017
|
+
except Unanswered as e:
|
|
1018
|
+
# the line is alive and the session stays; the screen may or may not have changed under the agent's list
|
|
1019
|
+
view.moved = True
|
|
1020
|
+
self.unsettled = True
|
|
1021
|
+
self._announce("agent", name, ok=False, code="timeout")
|
|
1022
|
+
raise Refused("timeout", f"{e}; the action may have happened: look before you retry.") from None
|
|
1023
|
+
except ConnectionLost as e:
|
|
1024
|
+
self.lost(e)
|
|
1025
|
+
if isinstance(e, AppNotAnswering): # the action went out, then the app stopped answering
|
|
1026
|
+
self._announce("agent", name, ok=False, code="not_answering")
|
|
1027
|
+
raise self._not_connected() from None
|
|
1028
|
+
self._announce("agent", name, ok=False, code="reconnecting")
|
|
1029
|
+
raise Refused("reconnecting", "the connection dropped during the action; look to check what "
|
|
1030
|
+
"happened") from None
|
|
1031
|
+
except Refused as e:
|
|
1032
|
+
# a touch a finger couldn't make: people watching see the step and why (VIEWER-PROTOCOL 1.4 `reason`)
|
|
1033
|
+
if e.code == "not_actionable":
|
|
1034
|
+
said = str(e)
|
|
1035
|
+
word = said.split(":", 1)[0] if ":" in said else ""
|
|
1036
|
+
self._announce("agent", name, label=element_name(said) or None, ok=False, code=e.code,
|
|
1037
|
+
reason=word if word.isidentifier() else None, caption=step_caption(name, said, args))
|
|
1038
|
+
raise
|
|
1039
|
+
if isinstance(result, Noted) and result.note:
|
|
1040
|
+
notes = [*notes, result.note]
|
|
1041
|
+
# The keyboard note is given only when the keyboard is still closed as the call answers (AGENT-TOOLS
|
|
1042
|
+
# v1.4): type_text focuses its input, so the keyboard is up again and the note would contradict the header
|
|
1043
|
+
closed_first = box.get("note") if name != "type_text" else None
|
|
1044
|
+
if closed_first:
|
|
1045
|
+
notes = [*notes, closed_first]
|
|
1046
|
+
if fresh[0]: # its own action keeps what it knew current; after someone else's, it must look
|
|
1047
|
+
view.seen = self.control.generation
|
|
1048
|
+
if isinstance(result, Unchanged): # nothing was done: nothing to settle or diff
|
|
1049
|
+
return _answer([str(result)], notes)
|
|
1050
|
+
reads_only = name == "alert" and args.get("action") == "text" # it changes nothing on the phone
|
|
1051
|
+
looks = settle is not None and args.get("look", True) and not reads_only
|
|
1052
|
+
view.moved = view.moved or not looks
|
|
1053
|
+
if not reads_only:
|
|
1054
|
+
self.unsettled = not looks
|
|
1055
|
+
self._announce("agent", name, label=element_name(str(result)), caption=step_caption(name, str(result), args),
|
|
1056
|
+
point=self._point(self.session.last_point))
|
|
1057
|
+
if not looks:
|
|
1058
|
+
view.unlooked += 0 if reads_only else 1
|
|
1059
|
+
taken = await self._taken_away(view, box.get("app"))
|
|
1060
|
+
self._recheck_later(client_id, box.get("app"), taken)
|
|
1061
|
+
return _answer([str(result)], [*notes, taken] if taken else notes)
|
|
1062
|
+
mark = box["mark"] # the stream as the gesture went out; the settle only trusts frames after the return
|
|
1063
|
+
# frames during the call count for actions with the normal cap; a launch may show a still splash first
|
|
1064
|
+
if isinstance(result, Toggled):
|
|
1065
|
+
landed = await self._toggle_landed(view, result)
|
|
1066
|
+
elif isinstance(result, Typed) and args.get("replace", True) and not args.get("submit"):
|
|
1067
|
+
landed = await self._typed_landed(view, result, args["text"])
|
|
1068
|
+
if landed is None and not result.field.secure and await self._typed_wrong(result, args["text"]) and \
|
|
1069
|
+
await self.control.agent(lambda: self.session.retype(args["text"])):
|
|
1070
|
+
# the field doesn't hold what was typed: keys went missing (iOS swapping in another keyboard as
|
|
1071
|
+
# the field took focus, seen on the phone 2026-11-11: "abc" landed as "a", and the retry with the
|
|
1072
|
+
# caret at the start made it "abca"). Cleared and typed once more, and the answer says so.
|
|
1073
|
+
notes = [*notes, NOTE_RETYPED]
|
|
1074
|
+
landed = await self._typed_landed(view, result, args["text"])
|
|
1075
|
+
else:
|
|
1076
|
+
landed = None
|
|
1077
|
+
if landed is not None:
|
|
1078
|
+
done = Settled(landed[1], True, False)
|
|
1079
|
+
else:
|
|
1080
|
+
done = await self._settle(self._mark()[0], cap=settle, since=mark[1],
|
|
1081
|
+
early=mark[0] if settle == CAP else None)
|
|
1082
|
+
if session_module.TRACE:
|
|
1083
|
+
self.log(f"trace {name} settle {done.seconds:.2f} s, changed={done.changed}")
|
|
1084
|
+
if done.capped:
|
|
1085
|
+
notes = [*notes, NOTE_CAPPED.format(cap=settle)]
|
|
1086
|
+
if name == "back" and not done.changed:
|
|
1087
|
+
raise Refused("not_actionable", "no_action: nothing changed after back" + self._still_open())
|
|
1088
|
+
# a toggle that changed nothing on the stream is not trusted: the tree says whether the switch flipped
|
|
1089
|
+
# Never on the AirPlay video's word alone: a mirror that stalls repeats its last picture, which looks exactly
|
|
1090
|
+
# like a still screen, so "No visible change" could be false. Then the tree read decides (0.16 s).
|
|
1091
|
+
if (not done.changed and name != "toggle" and not (full[0] or view.moved) and not view.shown_off
|
|
1092
|
+
and not self._fed_by_video()
|
|
1093
|
+
and (mark[1] is view.read_mark[1] or alike(mark[1], view.read_mark[1]))):
|
|
1094
|
+
# no frame changed after the call returned, and the picture at its start is the one the agent's
|
|
1095
|
+
# list was read against (nothing changed silently before it): no tree read, the list stays.
|
|
1096
|
+
# Not when something lies off screen: the action's effect may be there, and only a read shows it
|
|
1097
|
+
view.read_at, view.read_mark = self.control.clock(), self._mark()
|
|
1098
|
+
view.unlooked = 0 # this block covers them: nothing changed since the list
|
|
1099
|
+
return await self._add_shot(_answer([str(result), changes.SAME_SCREEN, changes.NO_CHANGE,
|
|
1100
|
+
settled_line(done.seconds)], notes), view, shot, at_read=False)
|
|
1101
|
+
to_target = name == "scroll" and args.get("to") == "target"
|
|
1102
|
+
# (a scroll to a target read the list after each of its drags, with the refs carried: its last read is no
|
|
1103
|
+
# move for the refs, though the content did move since the agent's last answer)
|
|
1104
|
+
moves = None if name in MAY_MOVE_CONTENT else name in MOVES_CONTENT and not to_target
|
|
1105
|
+
answer = await self._after(view, str(result), notes, full[0], moves, done.seconds, drifted=to_target,
|
|
1106
|
+
snap=landed[0] if landed is not None else None)
|
|
1107
|
+
if closed_first and view.read_keyboard: # the tap focused another input: the keyboard is up again
|
|
1108
|
+
answer["text"] = answer["text"].replace(f"\nNote: {closed_first}", "", 1)
|
|
1109
|
+
if isinstance(result, Toggled):
|
|
1110
|
+
# say only what the new list shows; if the switch can't be found again, claim nothing
|
|
1111
|
+
old = result.before
|
|
1112
|
+
same = [e for e in view.items if (e.role, e.label, e.identifier) == (old.role, old.label, old.identifier)]
|
|
1113
|
+
# by its ref; when the answer is a full list that numbered it anew (an agent's first call), by being the
|
|
1114
|
+
# only switch with that identity
|
|
1115
|
+
now = next((e for e in same if e.ref == result.ref), same[0] if len(same) == 1 else None)
|
|
1116
|
+
if now is not None:
|
|
1117
|
+
answer["text"] = answer["text"].replace(str(result), result.verdict(now, bool(view.alert)), 1)
|
|
1118
|
+
if isinstance(result, Typed): # the value as the new list shows it, not as it was typed
|
|
1119
|
+
old = result.field
|
|
1120
|
+
alike_fields = [e for e in view.items if e.role == old.role and (
|
|
1121
|
+
e.identifier == old.identifier if old.identifier else e.label == old.label)]
|
|
1122
|
+
now = next((e for e in view.items if result.same_field(e)),
|
|
1123
|
+
alike_fields[0] if len(alike_fields) == 1 else None) # renumbered in a first full list
|
|
1124
|
+
if now is not None:
|
|
1125
|
+
answer["text"] = answer["text"].replace(str(result), result.verdict(now), 1)
|
|
1126
|
+
taken = await self._taken_away(view, box.get("app"), read=self.session.app)
|
|
1127
|
+
self._recheck_later(client_id, box.get("app"), taken)
|
|
1128
|
+
if taken: # the screen the agent is shown is the app's, back in front, not the other app's
|
|
1129
|
+
return await self._look(view, shot, [str(result)], [*notes, taken])
|
|
1130
|
+
return await self._add_shot(answer, view, shot)
|
|
1131
|
+
|
|
1132
|
+
def _recheck_later(self, client_id: str, before: tuple[str, str] | None, taken: str | None) -> None:
|
|
1133
|
+
"""After a touch near the top that nothing was seen taking: the other app may come to the front a moment
|
|
1134
|
+
after the answer's check (seen on the phone 2026-11-11 with look: false), so the agent's next call asks
|
|
1135
|
+
again, if it comes within RECHECK_WITHIN and no person acted in between (Tapline's review)."""
|
|
1136
|
+
if taken is None and before is not None and self.session.top_touch:
|
|
1137
|
+
self._recheck[client_id] = (before, time.monotonic(), self.control.person_generation)
|
|
1138
|
+
else:
|
|
1139
|
+
self._recheck.pop(client_id, None)
|
|
1140
|
+
|
|
1141
|
+
async def _check_again(self, client_id: str, name: str) -> None:
|
|
1142
|
+
"""At an agent's call: the check _recheck_later left for it. Done before personal mode's own check of the
|
|
1143
|
+
app in front, so the agent hears that a banner took its touch rather than `not_allowed`. An action is
|
|
1144
|
+
refused `stale` (its refs are for the app, back in front now); a look carries the note. `apps` reads
|
|
1145
|
+
nothing of the screen and leaves the check for the next call; open_app makes it moot."""
|
|
1146
|
+
if name == "apps" or (pending := self._recheck.pop(client_id, None)) is None or name == "open_app":
|
|
1147
|
+
return
|
|
1148
|
+
before, when, persons = pending
|
|
1149
|
+
if time.monotonic() - when > RECHECK_WITHIN or self.control.person_generation != persons:
|
|
1150
|
+
return
|
|
1151
|
+
await self.wait_connected()
|
|
1152
|
+
view = self.control.client(client_id)
|
|
1153
|
+
taken = await self._taken_away(view, before, top=True)
|
|
1154
|
+
if taken is None:
|
|
1155
|
+
return
|
|
1156
|
+
if name in ACTIONS:
|
|
1157
|
+
view.moved = True
|
|
1158
|
+
raise Refused("stale", f"after your last action: {taken} This call did nothing.")
|
|
1159
|
+
self._taken_notes[client_id] = taken
|
|
1160
|
+
|
|
1161
|
+
async def _taken_away(self, view, before: tuple[str, str] | None,
|
|
1162
|
+
read: tuple[str, str] | None = None, top: bool | None = None) -> str | None:
|
|
1163
|
+
"""After an agent's touch that began where a notification banner comes down: did another app come to the
|
|
1164
|
+
front? A banner landing between the check before the touch and the touch itself takes it (seen on the phone
|
|
1165
|
+
2026-11-11: twice Mail opened, once on a toggle at y 159). Then the app the touch was meant for is opened
|
|
1166
|
+
again and the note says so; None when nothing took it. `read`: the app the answer's own read found in
|
|
1167
|
+
front, when it made one (then the phone is asked only if that differs); without one the phone is asked
|
|
1168
|
+
once (about 70 ms, and only after a touch near the top). The home screen or a notification pulled open
|
|
1169
|
+
count too, unless an alert is up (a permission prompt the touch raised is the app's own) or the phone's
|
|
1170
|
+
process list still has the app in front (a banner still sliding over it). `top`: whether the touch began
|
|
1171
|
+
near the top, when it isn't the latest action's (the check at the next call, see _run_tool)."""
|
|
1172
|
+
s = self.session
|
|
1173
|
+
if before is None or not (s.top_touch if top is None else top):
|
|
1174
|
+
return None
|
|
1175
|
+
name, bundle = before
|
|
1176
|
+
if bundle == SPRINGBOARD:
|
|
1177
|
+
# never the app a touch is meant for: a read taken while a banner slides over an app names SpringBoard
|
|
1178
|
+
# (seen on the phone 2026-11-11: "opened" the home screen over Settings), and a touch on the home
|
|
1179
|
+
# screen that opens an app is what it is for
|
|
1180
|
+
return None
|
|
1181
|
+
if read is not None and read[1] == bundle:
|
|
1182
|
+
return None
|
|
1183
|
+
front = await self._read(s.foreground_bundle_id)
|
|
1184
|
+
if not front or front == bundle:
|
|
1185
|
+
return None
|
|
1186
|
+
if await self._still_under(bundle):
|
|
1187
|
+
# the app still runs in front: something lies over it, not instead of it. Sign in with Apple, Apple Pay,
|
|
1188
|
+
# a Safari view, the share sheet are other processes over the app (Tapline's review), and a banner
|
|
1189
|
+
# slides over it. Only SpringBoard staying in front once a banner would have gone is a theft: a shade
|
|
1190
|
+
# (Notification Center, Control Center) pulled down from the top edge
|
|
1191
|
+
if front != SPRINGBOARD:
|
|
1192
|
+
return None
|
|
1193
|
+
if read is None: # the answer had no read of its own: one now, which waits out a banner
|
|
1194
|
+
await self._read(lambda: self._silent_read(view))
|
|
1195
|
+
if view.alert_now is not None or await self._read(s.foreground_bundle_id) != SPRINGBOARD:
|
|
1196
|
+
return None # a permission prompt the touch raised is the app's own; or the banner has gone
|
|
1197
|
+
elif front == SPRINGBOARD:
|
|
1198
|
+
if read is None:
|
|
1199
|
+
await self._read(lambda: self._silent_read(view))
|
|
1200
|
+
if view.alert_now is not None:
|
|
1201
|
+
return None
|
|
1202
|
+
await self.control.agent(lambda: s.launch(bundle))
|
|
1203
|
+
self.log(f"the agent's touch near the top left {bundle} for {front}: opened {bundle} again")
|
|
1204
|
+
if front == SPRINGBOARD:
|
|
1205
|
+
return NOTE_TAKEN_SYSTEM.format(other=SYSTEM_IN_FRONT, app=f"{name} ({bundle})")
|
|
1206
|
+
known = read[0] if read is not None and read[1] == front else s.app_names.get(front)
|
|
1207
|
+
return NOTE_TAKEN.format(other=f"{known or front} ({front})", app=f"{name} ({bundle})")
|
|
1208
|
+
|
|
1209
|
+
async def _still_under(self, bundle: str) -> bool:
|
|
1210
|
+
"""Whether iOS's process list still has the app running in front (under whatever WebDriverAgent names)."""
|
|
1211
|
+
try:
|
|
1212
|
+
return bundle in await asyncio.wait_for(self.session.device.foreground_apps(), 5)
|
|
1213
|
+
except Exception:
|
|
1214
|
+
return False # the phone can't say: count the app as gone, the safer side for an agent's next touch
|
|
1215
|
+
|
|
1216
|
+
async def turned(self, seq: int, jpeg: bytes, degrees: int) -> bytes:
|
|
1217
|
+
"""A stream frame turned upright for landscape, made once per frame for every viewer."""
|
|
1218
|
+
kept = self._turned
|
|
1219
|
+
if kept is not None and kept[0] == (seq, degrees):
|
|
1220
|
+
return kept[1]
|
|
1221
|
+
out = await asyncio.to_thread(turn_jpeg, jpeg, degrees)
|
|
1222
|
+
self._turned = ((seq, degrees), out)
|
|
1223
|
+
return out
|
|
1224
|
+
|
|
1225
|
+
async def _toggle_landed(self, view, result: "Toggled") -> tuple[Snapshot, float] | None:
|
|
1226
|
+
"""A switch's value changes the moment the touch lands; the knob's slide after it is only drawing. Waiting
|
|
1227
|
+
for the screen to stand still cost a toggle 0.33 to 0.65 s (measured on the phone 2026-11-11). So the tree
|
|
1228
|
+
is read until the switch shows its new value, and that read is the answer. Not when anything else
|
|
1229
|
+
appeared or went away in that read (a toggle that opens a section may still be animating it in): then
|
|
1230
|
+
None, and the usual wait for a still screen follows. None too when the switch doesn't flip in
|
|
1231
|
+
TOGGLE_LAND_WAIT."""
|
|
1232
|
+
loop = asyncio.get_running_loop()
|
|
1233
|
+
began = loop.time()
|
|
1234
|
+
old = result.before
|
|
1235
|
+
while loop.time() - began < TOGGLE_LAND_WAIT:
|
|
1236
|
+
snap = await self._read(lambda: self.session.snapshot())
|
|
1237
|
+
if snap.tree.alert is not None:
|
|
1238
|
+
return None # a confirm alert came up: the usual wait, and the verdict knows about alerts
|
|
1239
|
+
now = next((e for e in snap.tree.items if (e.role, e.identifier or "", e.label if not e.identifier else "")
|
|
1240
|
+
== (old.role, old.identifier or "", old.label if not old.identifier else "")), None)
|
|
1241
|
+
if now is not None and now.checked == result.flipped:
|
|
1242
|
+
if len(snap.tree.items) != len(view.items):
|
|
1243
|
+
return None
|
|
1244
|
+
return snap, loop.time() - began
|
|
1245
|
+
await asyncio.sleep(TOGGLE_POLL)
|
|
1246
|
+
return None
|
|
1247
|
+
|
|
1248
|
+
async def _typed_wrong(self, result: "Typed", text: str) -> bool:
|
|
1249
|
+
"""Does the field typed into hold something other than `text` on a fresh read? False when it can't be found
|
|
1250
|
+
again (claim nothing) or holds exactly `text`."""
|
|
1251
|
+
snap = await self._read(lambda: self.session.snapshot())
|
|
1252
|
+
old = result.field
|
|
1253
|
+
same = [e for e in snap.tree.items if e.role == old.role and (
|
|
1254
|
+
e.identifier == old.identifier if old.identifier else e.label == old.label)]
|
|
1255
|
+
return len(same) == 1 and not same[0].secure and same[0].value != text
|
|
1256
|
+
|
|
1257
|
+
async def _typed_landed(self, view, result: "Typed", text: str) -> tuple[Snapshot, float] | None:
|
|
1258
|
+
"""As _toggle_landed, for typing: the field holds the typed text as soon as the keys call returns, and the
|
|
1259
|
+
screen's wait after it (0.2 to 0.4 s on the phone) only saw the caret blink. The first read that shows the
|
|
1260
|
+
field with exactly that text is the answer, unless anything else appeared or went away in it."""
|
|
1261
|
+
loop = asyncio.get_running_loop()
|
|
1262
|
+
began = loop.time()
|
|
1263
|
+
old = result.field
|
|
1264
|
+
while loop.time() - began < TOGGLE_LAND_WAIT:
|
|
1265
|
+
snap = await self._read(lambda: self.session.snapshot())
|
|
1266
|
+
if snap.tree.alert is not None:
|
|
1267
|
+
return None
|
|
1268
|
+
same = [e for e in snap.tree.items if e.role == old.role and (
|
|
1269
|
+
e.identifier == old.identifier if old.identifier else e.label == old.label)]
|
|
1270
|
+
if len(same) == 1 and not same[0].secure and same[0].value == text:
|
|
1271
|
+
if len(snap.tree.items) != len(view.items):
|
|
1272
|
+
return None
|
|
1273
|
+
return snap, loop.time() - began
|
|
1274
|
+
await asyncio.sleep(TOGGLE_POLL)
|
|
1275
|
+
return None
|
|
1276
|
+
|
|
1277
|
+
async def _apps(self, notes: list) -> dict:
|
|
1278
|
+
apps = await self._read(self.session.apps)
|
|
1279
|
+
if not apps:
|
|
1280
|
+
return _answer(["No apps installed by you."], notes)
|
|
1281
|
+
try:
|
|
1282
|
+
front = await self._read(self.session.foreground_bundle_id)
|
|
1283
|
+
except Refused:
|
|
1284
|
+
raise
|
|
1285
|
+
except Exception:
|
|
1286
|
+
front = None # the list is still right without the marker
|
|
1287
|
+
lines = [app_line(a) + (" *" if a["bundle_id"] == front else "") for a in apps]
|
|
1288
|
+
return _answer([*lines, "* = in front."], notes)
|
|
1289
|
+
|
|
1290
|
+
async def _app(self, query: str, partial: bool = True) -> tuple[str, str]:
|
|
1291
|
+
"""(bundle id, how to show it) for what the agent named, in this order: an installed bundle id; a
|
|
1292
|
+
name, ignoring case, among the installed apps (exact, then, if `partial`, the start of a name); a
|
|
1293
|
+
built-in app's name; a bundle-id-looking string as given (built-in apps aren't in `apps`). Parts
|
|
1294
|
+
of bundle ids and of later words never match, so "Notes" doesn't hit "Sticky Notes"."""
|
|
1295
|
+
q = query.strip()
|
|
1296
|
+
apps = await self.session.apps()
|
|
1297
|
+
low = q.lower()
|
|
1298
|
+
for a in apps:
|
|
1299
|
+
if a["bundle_id"] == q:
|
|
1300
|
+
return q, f"{a['name']} ({q})"
|
|
1301
|
+
hits = [a for a in apps if a["name"].lower() == low]
|
|
1302
|
+
builtin = next((n for n in SYSTEM_APPS if n.lower() == low), None)
|
|
1303
|
+
if not hits and builtin:
|
|
1304
|
+
return SYSTEM_APPS[builtin], f"{builtin} ({SYSTEM_APPS[builtin]})"
|
|
1305
|
+
if not hits and partial:
|
|
1306
|
+
hits = [a for a in apps if a["name"].lower().startswith(low)]
|
|
1307
|
+
if not hits:
|
|
1308
|
+
if BUNDLE_ID.match(q):
|
|
1309
|
+
return q, q # launching it says whether it exists
|
|
1310
|
+
raise Refused("not_found", f'no installed app is named "{q}"; call apps to see them')
|
|
1311
|
+
if len(hits) > 1:
|
|
1312
|
+
raise Refused("ambiguous", f'{len(hits)} installed apps match "{q}". Pick one by its bundle id:\n'
|
|
1313
|
+
+ "\n".join(app_line(a) for a in hits[:10]))
|
|
1314
|
+
return hits[0]["bundle_id"], f"{hits[0]['name']} ({hits[0]['bundle_id']})"
|
|
1315
|
+
|
|
1316
|
+
async def _open_app(self, view, a: dict) -> str:
|
|
1317
|
+
bundle_id, shown = await self._app(a["app"])
|
|
1318
|
+
try:
|
|
1319
|
+
await self.session.launch(bundle_id)
|
|
1320
|
+
except ConnectionLost:
|
|
1321
|
+
raise
|
|
1322
|
+
except DeviceError as e:
|
|
1323
|
+
raise Refused("not_found", f"couldn't open {bundle_id}: {e}; call apps for the installed apps") from None
|
|
1324
|
+
return f"Opened {shown}."
|
|
1325
|
+
|
|
1326
|
+
async def _close_app(self, view, a: dict) -> str:
|
|
1327
|
+
bundle_id, shown = await self._app(a["app"], partial=False)
|
|
1328
|
+
await self.session.terminate(bundle_id)
|
|
1329
|
+
return f"Closed {shown}."
|
|
1330
|
+
|
|
1331
|
+
async def _close_stuck(self, stuck: AppNotAnswering, query: str, notes: list) -> dict:
|
|
1332
|
+
"""close_app while the app in front is frozen: end its processes through iOS (WebDriverAgent's own
|
|
1333
|
+
terminate would wait on the app), then connect again at once. Only that app: the rest can wait."""
|
|
1334
|
+
if self.control.paused:
|
|
1335
|
+
raise Refused("paused", "paused by the user in the live view; ask them to resume")
|
|
1336
|
+
if query.strip().lower() not in (stuck.bundle_id.lower(), stuck.name.lower()):
|
|
1337
|
+
raise self._not_connected()
|
|
1338
|
+
device = self.session.device
|
|
1339
|
+
if device.rsd is None:
|
|
1340
|
+
raise self._not_connected()
|
|
1341
|
+
await device.kill_app(stuck.bundle_id)
|
|
1342
|
+
self.log(f"closed {stuck.bundle_id}, which wasn't answering")
|
|
1343
|
+
self.stuck = None
|
|
1344
|
+
self._wake.set()
|
|
1345
|
+
self._announce("agent", "close_app", label=stuck.name)
|
|
1346
|
+
return _answer([f"Closed {stuck.name} ({stuck.bundle_id}), which wasn't answering."], notes)
|
|
1347
|
+
|
|
1348
|
+
async def _wait(self, view, args: dict, notes: list, shot) -> dict:
|
|
1349
|
+
"""Sleep outside the phone's queue (a person may act meanwhile), then answer with what changed."""
|
|
1350
|
+
seconds = args["seconds"]
|
|
1351
|
+
await self.control.sleep(seconds)
|
|
1352
|
+
result = f"Waited {seconds:.1f} s."
|
|
1353
|
+
if not args.get("look", True):
|
|
1354
|
+
view.unlooked += 1
|
|
1355
|
+
return _answer([result], notes)
|
|
1356
|
+
full = view.seen != self.control.generation or view.dropped or view.read_at == float("-inf")
|
|
1357
|
+
answer = await self._after(view, result, notes, full) # no `Settled in`: the app did nothing
|
|
1358
|
+
return await self._add_shot(answer, view, shot)
|
|
1359
|
+
|
|
1360
|
+
async def _wait_for(self, view, args: dict, notes: list, shot) -> dict:
|
|
1361
|
+
"""Poll the fast tree (outside the phone's queue between reads) until the text, id or label is
|
|
1362
|
+
on the screen (or, with `gone`, no longer is), then answer with what changed. No `Settled in`."""
|
|
1363
|
+
kind = next(k for k in WAIT_FOR_KINDS if args.get(k) is not None)
|
|
1364
|
+
target = ui.parse_target({kind: args[kind]})
|
|
1365
|
+
gone, timeout = bool(args.get("gone", False)), float(args.get("timeout", WAIT_FOR_DEFAULT))
|
|
1366
|
+
clock = self.control.clock
|
|
1367
|
+
began = clock()
|
|
1368
|
+
timed_out = Refused("timeout", f'Waited {timeout:g} s for {kind} "{args[kind]}" to '
|
|
1369
|
+
f"{'disappear' if gone else 'appear'}; it didn't happen.")
|
|
1370
|
+
seen = False # has the screen been read at all during this wait?
|
|
1371
|
+
while True:
|
|
1372
|
+
left = timeout - (clock() - began)
|
|
1373
|
+
if left < WAIT_FOR_MIN_LEFT:
|
|
1374
|
+
raise timed_out
|
|
1375
|
+
# The read gets its own full deadline, so a slow phone isn't taken for a dead one; wait_for's
|
|
1376
|
+
# deadline is kept by cancelling the read, which answers `timeout` and leaves the link alone.
|
|
1377
|
+
# "It didn't happen" needs one look at the screen, so the first read gets at least
|
|
1378
|
+
# WAIT_FOR_FIRST_READ seconds, whatever the wait's own time: over Wi-Fi one read can take most of a
|
|
1379
|
+
# second, and a 0.5 s wait then never saw the screen at all (seen 2026-11-10: twelve of twelve). If
|
|
1380
|
+
# even that read doesn't come back (a stall), the answer says that the screen wasn't read, not what
|
|
1381
|
+
# was on it (the same day: a 5 s wait said a text didn't appear which had been there all along).
|
|
1382
|
+
grace = left if seen else max(left, WAIT_FOR_FIRST_READ)
|
|
1383
|
+
try:
|
|
1384
|
+
tree = await asyncio.wait_for(
|
|
1385
|
+
self._read(lambda: self.session.refresh(timeout=WAIT_FOR_READ_MAX)), grace)
|
|
1386
|
+
except asyncio.TimeoutError:
|
|
1387
|
+
if not seen:
|
|
1388
|
+
raise Refused("timeout", f"The phone didn't answer within {max(timeout, WAIT_FOR_FIRST_READ):g} s, so it "
|
|
1389
|
+
f'isn\'t known whether {kind} "{args[kind]}" is on screen; look, then '
|
|
1390
|
+
"wait again.") from None
|
|
1391
|
+
raise timed_out from None
|
|
1392
|
+
seen = True
|
|
1393
|
+
if self._missing(tree.items, target) == gone:
|
|
1394
|
+
break
|
|
1395
|
+
left = timeout - (clock() - began)
|
|
1396
|
+
if left < WAIT_FOR_MIN_LEFT:
|
|
1397
|
+
raise timed_out
|
|
1398
|
+
await self.control.sleep(min(WAIT_FOR_POLL, left))
|
|
1399
|
+
result = f"Done after {clock() - began:.1f} s."
|
|
1400
|
+
if not args.get("look", True):
|
|
1401
|
+
view.unlooked += 1
|
|
1402
|
+
return _answer([result], notes)
|
|
1403
|
+
full = view.seen != self.control.generation or view.dropped or view.read_at == float("-inf")
|
|
1404
|
+
# the last poll is the answer's tree: no read of its own, only the app lookup the polls left out
|
|
1405
|
+
snap = await self._read(lambda: self.session.snapshot_from(tree))
|
|
1406
|
+
return await self._add_shot(await self._after(view, result, notes, full, snap=snap), view, shot)
|
|
1407
|
+
|
|
1408
|
+
async def _type_text(self, view, a: dict) -> Typed:
|
|
1409
|
+
"""Type into the target, or into the only input on screen (the fast tree has no focus attribute).
|
|
1410
|
+
The target is tapped first by the session, which waits for the keyboard and for focus to arrive."""
|
|
1411
|
+
target = a.get("target")
|
|
1412
|
+
if target is None:
|
|
1413
|
+
await self._silent_read(view) # which inputs there are must be read now
|
|
1414
|
+
inputs = [e for e in view.items if e.role == "input"]
|
|
1415
|
+
if not inputs:
|
|
1416
|
+
raise Refused("not_found", "No input on screen")
|
|
1417
|
+
if len(inputs) > 1:
|
|
1418
|
+
shown = "\n".join([*(f" {e.line()}" for e in inputs[:ui.MAX_CANDIDATES]),
|
|
1419
|
+
*ui.more_candidates(len(inputs), ui.MAX_CANDIDATES),
|
|
1420
|
+
"Retry with one of the refs as the target."]) # no target, so no `index` to add
|
|
1421
|
+
raise Refused("ambiguous", f"{len(inputs)} inputs on screen; pass a target to pick one:\n"
|
|
1422
|
+
f"Candidates:\n{shown}")
|
|
1423
|
+
field = inputs[0]
|
|
1424
|
+
elif target.point is not None:
|
|
1425
|
+
x, y = target.point
|
|
1426
|
+
field = ui.at_point(view.items, target.point, "input")
|
|
1427
|
+
if field is None:
|
|
1428
|
+
raise Refused("not_actionable", f"no_action: Nothing at ({round(x)}, {round(y)}) takes typing.")
|
|
1429
|
+
else:
|
|
1430
|
+
field = ui.resolve(view.items, target)
|
|
1431
|
+
if field.role != "input":
|
|
1432
|
+
raise Refused("not_actionable", f"no_action: {field.line(actions=False)} is not an input")
|
|
1433
|
+
# the focusing tap is a touch: refused where tap would be, and made where it shows; `force` as for tap
|
|
1434
|
+
self._refuse_disabled(field)
|
|
1435
|
+
forced, at = "", None
|
|
1436
|
+
if a.get("force"):
|
|
1437
|
+
forced = self._forced(view, field)
|
|
1438
|
+
else:
|
|
1439
|
+
self._guard(view, field)
|
|
1440
|
+
at = self._touch_point(field)
|
|
1441
|
+
async def typed() -> None:
|
|
1442
|
+
await self.session.type_text(a["text"], replace=bool(a.get("replace", True)),
|
|
1443
|
+
submit=bool(a.get("submit", False)), target=ui.Target(ref=field.ref), at=at)
|
|
1444
|
+
try:
|
|
1445
|
+
await typed()
|
|
1446
|
+
except Refused as e:
|
|
1447
|
+
# The keyboard was left up for the tap on another input, and this app gave that tap to the keyboard
|
|
1448
|
+
# (it closed it, or nothing happened): close it the old way and tap again. Nothing was typed.
|
|
1449
|
+
if not (view.read_keyboard and "didn't take focus" in str(e)):
|
|
1450
|
+
raise
|
|
1451
|
+
await self._close_keyboard(view, "type_text", ui.Target(ref=field.ref), shown_inputs=True)
|
|
1452
|
+
await typed()
|
|
1453
|
+
done = Typed(f"Typed into {field.line(actions=False)}.")
|
|
1454
|
+
retyped = NOTE_RETYPED if getattr(self.session, "retyped", False) else ""
|
|
1455
|
+
done.field, done.note = field, " ".join(n for n in (forced, retyped) if n)
|
|
1456
|
+
return done
|
|
1457
|
+
|
|
1458
|
+
async def _scroll(self, view, a: dict) -> Scrolled:
|
|
1459
|
+
"""Swipe through the centre of the scroll area (the innermost one around the target, the point or the
|
|
1460
|
+
screen's centre), `amount` of its height (or width), between the bars. `to` repeats that until a swipe
|
|
1461
|
+
leaves the settled screen as the one before left it (a rubber band or a spinner moves the screen
|
|
1462
|
+
meanwhile, so "no change at all" would never come), at most MAX_SCROLL_SWIPES times or SCROLL_CAP seconds."""
|
|
1463
|
+
s = self.session
|
|
1464
|
+
target, to = a.get("target"), a.get("to")
|
|
1465
|
+
if to == "target":
|
|
1466
|
+
return await self._scroll_to(view, target)
|
|
1467
|
+
if target is None:
|
|
1468
|
+
area = ui.scroll_container(view.items, s.screen)
|
|
1469
|
+
elif target.point is not None:
|
|
1470
|
+
x, y = target.point
|
|
1471
|
+
area = ui.scroll_at(view.items, target.point)
|
|
1472
|
+
if area is None:
|
|
1473
|
+
raise Refused("not_actionable", f"no_action: Nothing at ({round(x)}, {round(y)}) takes a scroll.")
|
|
1474
|
+
else:
|
|
1475
|
+
hit = ui.resolve(view.items, target)
|
|
1476
|
+
area = ui.scroll_of(view.items, hit)
|
|
1477
|
+
if area is None:
|
|
1478
|
+
raise Refused("not_actionable", f"no_action: {hit.line(actions=False)} is not inside a scroll view")
|
|
1479
|
+
direction = a.get("direction") or (SCROLL_TO[to][0] if to else "down") # _checked made `to` and it agree
|
|
1480
|
+
top, bottom = s.last_insets
|
|
1481
|
+
band = (SCROLL_TOP_INSET if top is None else top,
|
|
1482
|
+
s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom))
|
|
1483
|
+
x0, y0, x1, y1 = ui.scroll_area_points(direction, area.rect if area else (0.0, 0.0, *s.screen),
|
|
1484
|
+
a.get("amount", 0.8), band)
|
|
1485
|
+
bounds = area.rect if area else (0.0, 0.0, *s.screen)
|
|
1486
|
+
# Whether the tree shows what lies past the list's edge. Where it does (React Native renders a whole list),
|
|
1487
|
+
# the end is reached when a read after a drag shows nothing past it any more: no waiting for the pictures
|
|
1488
|
+
# to repeat, which over Wi-Fi took 4 to 14 s. A list that keeps only its visible rows shows nothing past
|
|
1489
|
+
# its edge either way: it is scrolled as before, until the pictures stop changing.
|
|
1490
|
+
beyond = bool(to) and direction in ("up", "down") and self._more_beyond(direction, bounds)
|
|
1491
|
+
other = {"up": "down", "down": "up"}.get(direction)
|
|
1492
|
+
if to and other and not beyond and self._more_beyond(other, bounds):
|
|
1493
|
+
# The tree shows this list's rows past its other edge, so it would show them past this one too: there
|
|
1494
|
+
# are none, so the list is already at its start (or end). No drag, and no wait for pictures to repeat
|
|
1495
|
+
# (13 s over Wi-Fi, 2026-11-11). (A list that keeps only its visible rows shows none either way and
|
|
1496
|
+
# is scrolled as before.)
|
|
1497
|
+
return Scrolled(f"{area.line(actions=False) if area else 'The screen'} is already at its {to}.")
|
|
1498
|
+
loop = asyncio.get_running_loop()
|
|
1499
|
+
began, previous, note = loop.time(), self._mark()[1], ""
|
|
1500
|
+
for swipes in range(1, MAX_SCROLL_SWIPES + 1):
|
|
1501
|
+
seq, since = self._mark()
|
|
1502
|
+
await s.scroll(x0, y0, x1, y1)
|
|
1503
|
+
if not to or swipes == MAX_SCROLL_SWIPES:
|
|
1504
|
+
break
|
|
1505
|
+
await self._settle(seq, cap=RESOLVE_SETTLE, since=since)
|
|
1506
|
+
settled = self._mark()[1]
|
|
1507
|
+
if beyond:
|
|
1508
|
+
await self._silent_read(view)
|
|
1509
|
+
if not self._more_beyond(direction, bounds):
|
|
1510
|
+
break # the tree said there was more past that edge, and now there is none: reached
|
|
1511
|
+
if alike(settled, previous):
|
|
1512
|
+
break # the swipe left the screen as the one before did: the end of the list
|
|
1513
|
+
previous = settled # (with no stream there is nothing to compare: every swipe is made)
|
|
1514
|
+
if loop.time() - began >= SCROLL_CAP:
|
|
1515
|
+
# the screen still changed after every swipe: more may be loading, or the list is long
|
|
1516
|
+
note = f"the {to} wasn't reached in {SCROLL_CAP:g} s; scroll again to go on."
|
|
1517
|
+
break
|
|
1518
|
+
done = Scrolled(f"Scrolled {area.line(actions=False) if area else 'the screen'} {direction}.")
|
|
1519
|
+
done.note = note
|
|
1520
|
+
return done
|
|
1521
|
+
|
|
1522
|
+
def _more_beyond(self, direction: str, bounds: tuple[float, float, float, float]) -> bool:
|
|
1523
|
+
"""Does the last read have elements past the scroll area's top (`up`) or bottom (`down`) edge, within its
|
|
1524
|
+
width? What a list holds beyond its edge is in the tree (React Native renders it, UIKit keeps nearby rows);
|
|
1525
|
+
an empty answer means its start (or end) is on screen."""
|
|
1526
|
+
x, y, w, h = bounds
|
|
1527
|
+
for e in getattr(self.session, "offscreen", []):
|
|
1528
|
+
if e.x + e.width <= x or e.x >= x + w:
|
|
1529
|
+
continue # beside the area (a carousel's cards), not past its edge
|
|
1530
|
+
# by its middle: a row that straddles the edge with its middle past it is still past it (seen over
|
|
1531
|
+
# Wi-Fi 2026-11-11: Bottom at y 915 to 963 on a 932 pt screen, and the scroll stopped one drag short)
|
|
1532
|
+
if direction == "up" and e.center[1] < y:
|
|
1533
|
+
return True
|
|
1534
|
+
if direction == "down" and e.center[1] >= y + h:
|
|
1535
|
+
return True
|
|
1536
|
+
return False
|
|
1537
|
+
|
|
1538
|
+
def _anywhere(self, view, target: ui.Target) -> ui.Element:
|
|
1539
|
+
"""The element a target names, on the agent's list or, failing that, among what the last read found off
|
|
1540
|
+
screen. Raises as `resolve` does when it is in neither."""
|
|
1541
|
+
off = getattr(self.session, "offscreen", [])
|
|
1542
|
+
if target.text is not None:
|
|
1543
|
+
# {text} is an exact match first, then a part of one: an element off screen that is exactly the text
|
|
1544
|
+
# comes before one on screen that only contains it (seen on the phone 2026-11-10: "Alert" found the
|
|
1545
|
+
# state line, which reads "… alert=none …", and the Alert button below the screen was never reached)
|
|
1546
|
+
want = ui._norm(target.text)
|
|
1547
|
+
if (not any(want in (ui._norm(e.label), ui._norm(e.value)) for e in view.items)
|
|
1548
|
+
and any(want in (ui._norm(e.label), ui._norm(e.value)) for e in off)):
|
|
1549
|
+
return ui.resolve(sorted([*view.items, *off], key=lambda e: (e.y, e.x)), target)
|
|
1550
|
+
try:
|
|
1551
|
+
return ui.resolve(view.items, target)
|
|
1552
|
+
except Refused as e:
|
|
1553
|
+
if e.code != "not_found" or target.ref is not None:
|
|
1554
|
+
raise
|
|
1555
|
+
# on screen and off it together, in reading order: an `index` counts through both, and several
|
|
1556
|
+
# matches without one are `ambiguous` (the first off-screen one may lie the other way)
|
|
1557
|
+
everything = sorted([*view.items, *off], key=lambda e: (e.y, e.x))
|
|
1558
|
+
|
|
1559
|
+
return ui.resolve(everything, target)
|
|
1560
|
+
|
|
1561
|
+
async def _scroll_to(self, view, target: ui.Target) -> Scrolled:
|
|
1562
|
+
"""`scroll` with `to: "target"` (AGENT-TOOLS v1.6): drag the scroll view that holds the element until the
|
|
1563
|
+
element is on screen and clear of the bars. The tree gives the place of an element that is off screen, and
|
|
1564
|
+
an eased drag doesn't slide on, so each drag covers a known distance; the screen is read after each one,
|
|
1565
|
+
and the next drag covers what is left. It stops when the element is clear, when a drag no longer moves it
|
|
1566
|
+
(the end of the list), or at the limits `to: "end"` has. A row a lazy list hasn't made yet is `not_found`."""
|
|
1567
|
+
s = self.session
|
|
1568
|
+
loop = asyncio.get_running_loop()
|
|
1569
|
+
began, note, where, clear = loop.time(), "", None, False
|
|
1570
|
+
seen: list[tuple[float, ...]] = [] # where the element was before each drag
|
|
1571
|
+
for drags in range(MAX_SCROLL_SWIPES + 1):
|
|
1572
|
+
try:
|
|
1573
|
+
element = self._anywhere(view, target)
|
|
1574
|
+
except Refused:
|
|
1575
|
+
if seen: # the screen has moved under the agent's list, and this answer carries none
|
|
1576
|
+
view.moved = True
|
|
1577
|
+
raise
|
|
1578
|
+
area = ui.scroll_holding(view.items, element, s.screen)
|
|
1579
|
+
where = where or (area.line(actions=False) if area else "the screen")
|
|
1580
|
+
top, bottom = s.last_insets
|
|
1581
|
+
band = (SCROLL_TOP_INSET if top is None else top,
|
|
1582
|
+
s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom))
|
|
1583
|
+
x, y, w, h = area.rect if area else (0.0, 0.0, *s.screen)
|
|
1584
|
+
y, h = max(y, band[0]), min(y + h, band[1]) - max(y, band[0])
|
|
1585
|
+
if h <= 0: # the scroll view itself lies under the bars: nothing to drag in
|
|
1586
|
+
note = "it doesn't scroll further: the target is as far into view as it gets."
|
|
1587
|
+
break
|
|
1588
|
+
move = ui.scroll_move(element.rect, (x, y, w, h))
|
|
1589
|
+
# after a drag, inside the view is enough: chasing the last points of margin with a drag that can't
|
|
1590
|
+
# be shorter than SCROLL_MIN_DRAG would go back and forth
|
|
1591
|
+
if move == (0.0, 0.0) or (seen and ui.scroll_move(element.rect, (x, y, w, h), margin=0.0) == (0.0, 0.0)):
|
|
1592
|
+
clear = True
|
|
1593
|
+
break
|
|
1594
|
+
if any(ui._same_rect(was, element.rect) for was in seen):
|
|
1595
|
+
note = "it doesn't scroll further: the target is as far into view as it gets."
|
|
1596
|
+
break
|
|
1597
|
+
if loop.time() - began >= SCROLL_CAP:
|
|
1598
|
+
note = f"the target wasn't reached in {SCROLL_CAP:g} s; scroll again to go on."
|
|
1599
|
+
break
|
|
1600
|
+
if drags == MAX_SCROLL_SWIPES:
|
|
1601
|
+
note = f"the target wasn't reached in {MAX_SCROLL_SWIPES} drags; scroll again to go on."
|
|
1602
|
+
break
|
|
1603
|
+
seen.append(element.rect)
|
|
1604
|
+
await s.scroll(*ui.scroll_step(move, (x, y, w, h)))
|
|
1605
|
+
await self._silent_read(view) # once the screen is still: where the element is now
|
|
1606
|
+
line = element.line(ref=bool(element.ref), actions=False)
|
|
1607
|
+
if clear and not seen: # nothing was dragged: "Scrolled" would read as if the layout had moved
|
|
1608
|
+
return Scrolled(f"{line} is already on screen.")
|
|
1609
|
+
done = Scrolled(f"Scrolled {where} to {line}." if clear else f"Scrolled {where} toward {line}.")
|
|
1610
|
+
done.note = note
|
|
1611
|
+
return done
|
|
1612
|
+
|
|
1613
|
+
async def _outside_focus(self, view, name: str, target: ui.Target, keyboard_top: float,
|
|
1614
|
+
shown_inputs: bool = False) -> bool:
|
|
1615
|
+
"""Is the touch's target something other than the focused input, so the keyboard is in the way of it
|
|
1616
|
+
(AGENT-TOOLS v1.4)? False for the focused input itself, an element in an OS alert, a point on the
|
|
1617
|
+
keyboard (a key), and whatever the tool is about to refuse (nothing is touched then). False too for
|
|
1618
|
+
`type_text` into another input that shows above the keyboard: a tap there moves the focus, as a person's
|
|
1619
|
+
does when filling a form, and closing the keyboard between two fields cost two seconds a field (measured
|
|
1620
|
+
on the phone 2026-11-10: 3.4 s against 1.7 s). `shown_inputs` asks about those as well."""
|
|
1621
|
+
items = view.items
|
|
1622
|
+
if target.point is not None:
|
|
1623
|
+
x, y = target.point
|
|
1624
|
+
if name != "type_text" and y >= keyboard_top:
|
|
1625
|
+
return False
|
|
1626
|
+
if any(e.in_alert and e.x <= x < e.x + e.width and e.y <= y < e.y + e.height for e in items):
|
|
1627
|
+
return False
|
|
1628
|
+
field = ui.at_point(items, target.point, "input")
|
|
1629
|
+
else:
|
|
1630
|
+
try:
|
|
1631
|
+
field = ui.resolve(items, target)
|
|
1632
|
+
except Refused:
|
|
1633
|
+
return False
|
|
1634
|
+
if isinstance(field, tuple) or field.in_alert:
|
|
1635
|
+
return False
|
|
1636
|
+
if name == "type_text" and (field is None or field.role != "input"):
|
|
1637
|
+
return False # it refuses typing into this itself
|
|
1638
|
+
if name == "type_text" and not shown_inputs and keyboard_top - field.y >= MIN_VISIBLE:
|
|
1639
|
+
# at least 10 pt of it show above the keyboard: a person taps that part and focus moves (AGENT-TOOLS
|
|
1640
|
+
# §4, v1.6; the lab's second field shows 17 pt, and closing first cost 2 s there)
|
|
1641
|
+
return False
|
|
1642
|
+
return field is None or not await self.session.is_focused(field) # a non-input is outside by definition
|
|
1643
|
+
|
|
1644
|
+
async def _close_keyboard(self, view, name: str, target: ui.Target | None, shown_inputs: bool = False) -> str:
|
|
1645
|
+
"""Before a touch (or a focusing tap) on something other than the focused input while the keyboard is up:
|
|
1646
|
+
close it the way a person does, since the first tap outside the field only closes it and the keyboard
|
|
1647
|
+
covers what is near the bottom. One touch where nothing responds, then WebDriverAgent's own dismissal
|
|
1648
|
+
if it is still up; neither presses return. Gives the note when it ended up closed, else leaves the
|
|
1649
|
+
screen to the usual `covered` rules."""
|
|
1650
|
+
s = self.session
|
|
1651
|
+
if name not in KEYBOARD_TOOLS or target is None or not view.read_keyboard or view.alert_now is not None:
|
|
1652
|
+
return "" # (an OS alert in front is modal: a touch outside it closes nothing, and behind_alert decides)
|
|
1653
|
+
top, bottom = s.last_insets
|
|
1654
|
+
top = SCROLL_TOP_INSET if top is None else top
|
|
1655
|
+
bottom = s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom)
|
|
1656
|
+
if not await self._outside_focus(view, name, target, bottom, shown_inputs):
|
|
1657
|
+
return ""
|
|
1658
|
+
return await self._dismiss_keyboard_now(view)
|
|
1659
|
+
|
|
1660
|
+
async def _dismiss_keyboard_now(self, view) -> str:
|
|
1661
|
+
"""Close the keyboard the way a person does: one touch where nothing responds, then WebDriverAgent's own
|
|
1662
|
+
dismissal if it is still up; neither presses return. NOTE_KEYBOARD when it ended up closed, else ""."""
|
|
1663
|
+
s = self.session
|
|
1664
|
+
top, bottom = s.last_insets
|
|
1665
|
+
top = SCROLL_TOP_INSET if top is None else top
|
|
1666
|
+
bottom = s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom)
|
|
1667
|
+
if (blank := ui.blank_point(view.items, s.screen, top, bottom)) is not None:
|
|
1668
|
+
await s.tap(ui.Target(x=blank[0], y=blank[1]))
|
|
1669
|
+
await self._silent_read(view)
|
|
1670
|
+
if view.read_keyboard and await s.dismiss_keyboard():
|
|
1671
|
+
await self._silent_read(view)
|
|
1672
|
+
return "" if view.read_keyboard else NOTE_KEYBOARD
|
|
1673
|
+
|
|
1674
|
+
def _under_keyboard(self, target: ui.Target) -> bool:
|
|
1675
|
+
"""Is the target an element the last read found past its scroll view's edge but still on the phone's
|
|
1676
|
+
screen: hidden by the keyboard rather than scrolled away?"""
|
|
1677
|
+
try:
|
|
1678
|
+
hit = ui.resolve(getattr(self.session, "offscreen", []), dataclasses.replace(target, index=0))
|
|
1679
|
+
except Refused:
|
|
1680
|
+
return False
|
|
1681
|
+
return isinstance(hit, ui.Element) and 0 <= hit.y < self.session.screen[1] and hit.y + hit.height > 0
|
|
1682
|
+
|
|
1683
|
+
@staticmethod
|
|
1684
|
+
def _refuse_behind_alert(view, force: bool = True) -> None:
|
|
1685
|
+
"""When the newest read shows an OS alert: `behind_alert`. `force` says whether passing it could help
|
|
1686
|
+
(it can't touch an element that isn't found)."""
|
|
1687
|
+
if view.alert_now is not None:
|
|
1688
|
+
title, buttons = view.alert_now
|
|
1689
|
+
alert = " ".join([json.dumps(title, ensure_ascii=False), *(f"[{b}]" for b in buttons)])
|
|
1690
|
+
raise Refused("not_actionable", f"behind_alert: an alert covers it: {alert}; answer it with alert, "
|
|
1691
|
+
+ ("or pass force: true." if force else "or ask the user."))
|
|
1692
|
+
|
|
1693
|
+
def _refuse_behind_sheet(self, target: ui.Target) -> None:
|
|
1694
|
+
"""A target on no listed element but on one an open sheet (or another layer) hides: `covered`, naming what
|
|
1695
|
+
is in front, not `not_found`. It isn't on the list, so `force` has nothing to touch."""
|
|
1696
|
+
behind = getattr(self.session, "behind", [])
|
|
1697
|
+
try:
|
|
1698
|
+
hit = ui.resolve([e for e, _ in behind], dataclasses.replace(target, index=0))
|
|
1699
|
+
except Refused:
|
|
1700
|
+
return
|
|
1701
|
+
for e, front in behind:
|
|
1702
|
+
if e is hit:
|
|
1703
|
+
raise Refused("not_actionable", f"covered: {hit.line(ref=False, actions=False)} is behind "
|
|
1704
|
+
f"{self._front_line(front)}; close that first.")
|
|
1705
|
+
|
|
1706
|
+
def _front_line(self, front: ui.Element | None) -> str:
|
|
1707
|
+
"""The line of what is open in front, with the ref the agent's list has for it (the read that found it
|
|
1708
|
+
numbered it for itself); with no ref when the agent's list doesn't have it."""
|
|
1709
|
+
if front is None:
|
|
1710
|
+
return "what is open in front of it"
|
|
1711
|
+
plain = front.line(ref=False, actions=False)
|
|
1712
|
+
known = next((e for e in self.session.items if e.line(ref=False, actions=False) == plain), None)
|
|
1713
|
+
return known.line(actions=False) if known is not None else plain
|
|
1714
|
+
|
|
1715
|
+
def _still_open(self) -> str:
|
|
1716
|
+
"""For a `back` that changed nothing: what is open on top, if the last read found a layer in front."""
|
|
1717
|
+
behind = getattr(self.session, "behind", [])
|
|
1718
|
+
if not behind:
|
|
1719
|
+
return "."
|
|
1720
|
+
front = behind[0][1]
|
|
1721
|
+
named = self._front_line(front) if front is not None else "a sheet or menu"
|
|
1722
|
+
return f"; {named} is still open: tap its close control."
|
|
1723
|
+
|
|
1724
|
+
def _refuse_off_screen(self, target: ui.Target) -> None:
|
|
1725
|
+
"""A target on no listed element but on one the last read found past the screen (or its scroll view's
|
|
1726
|
+
edge): `off_screen`, not `not_found`. A touch can't reach it, so `force` doesn't either."""
|
|
1727
|
+
try:
|
|
1728
|
+
hit = ui.resolve(getattr(self.session, "offscreen", []), dataclasses.replace(target, index=0))
|
|
1729
|
+
except Refused:
|
|
1730
|
+
return
|
|
1731
|
+
if isinstance(hit, ui.Element):
|
|
1732
|
+
raise Refused("not_actionable", f"off_screen: {hit.line(ref=False, actions=False)} is off screen; "
|
|
1733
|
+
f"{SCROLL_HINT}")
|
|
1734
|
+
|
|
1735
|
+
@staticmethod
|
|
1736
|
+
def _refuse_disabled(element: ui.Element) -> None:
|
|
1737
|
+
if not element.enabled:
|
|
1738
|
+
raise Refused("not_actionable", f"disabled: {element.line(actions=False)} is disabled.")
|
|
1739
|
+
|
|
1740
|
+
def _guard(self, view, element: ui.Element) -> None:
|
|
1741
|
+
"""A9: refuse what a real touch couldn't do. The reason word comes first in the message."""
|
|
1742
|
+
self._refuse_disabled(element)
|
|
1743
|
+
if not element.in_alert:
|
|
1744
|
+
self._refuse_behind_alert(view)
|
|
1745
|
+
|
|
1746
|
+
def _touch_point(self, element: ui.Element) -> tuple[float, float] | None:
|
|
1747
|
+
"""Where a finger would touch an element a bar (or the keyboard) is drawn over its centre: the middle of
|
|
1748
|
+
the part that shows after every overlapping bar is cut away, nearest the centre. None when no bar
|
|
1749
|
+
covers the centre. Refuses when too little shows."""
|
|
1750
|
+
centre = element.center
|
|
1751
|
+
bars = element.covered_by
|
|
1752
|
+
width, height = self.session.screen or (0.0, 0.0)
|
|
1753
|
+
if height > width and element.rect[1] < STATUS_BAR:
|
|
1754
|
+
bars = (*bars, ui.Bar("StatusBar", (0.0, 0.0, width, STATUS_BAR), "the status bar"))
|
|
1755
|
+
over = [b for b in bars if b.rect[0] <= centre[0] < b.rect[0] + b.rect[2]
|
|
1756
|
+
and b.rect[1] <= centre[1] < b.rect[1] + b.rect[3]]
|
|
1757
|
+
if not over:
|
|
1758
|
+
return None
|
|
1759
|
+
x, y, w, h = ui.visible_part(element.rect, bars, self.session.screen)
|
|
1760
|
+
if w >= MIN_VISIBLE and h >= MIN_VISIBLE:
|
|
1761
|
+
return x + w / 2, y + h / 2
|
|
1762
|
+
line = element.line(actions=False)
|
|
1763
|
+
if over[0].kind == "StatusBar":
|
|
1764
|
+
raise Refused("not_actionable", f"covered: {line} is under the status bar, where a touch scrolls the "
|
|
1765
|
+
f"list to its top; {SCROLL_HINT}")
|
|
1766
|
+
if over[0].kind == "TabBar":
|
|
1767
|
+
raise Refused("not_actionable", f"under_tab_bar: {line} is under the tab bar iOS draws; {SCROLL_HINT}")
|
|
1768
|
+
if over[0].kind in ui.COVERING_BARS:
|
|
1769
|
+
raise Refused("not_actionable", f"covered: {line} is covered by {over[0].line}; {SCROLL_HINT}")
|
|
1770
|
+
raise Refused("not_actionable", f"covered: {line} is covered by {over[0].line}; move or close it first.")
|
|
1771
|
+
|
|
1772
|
+
async def _touch(self, view, a: dict, hold_ms: int) -> str:
|
|
1773
|
+
"""`tap` and `long_press`: refuse a disabled element or one behind an alert, and touch the part of an
|
|
1774
|
+
element under a bar that shows. `force` skips all of it. A point is a raw touch."""
|
|
1775
|
+
s, target = self.session, _target(a)
|
|
1776
|
+
verb = "Long-pressed" if hold_ms > 300 else "Tapped"
|
|
1777
|
+
found = ui.resolve(s.items, target)
|
|
1778
|
+
if isinstance(found, tuple):
|
|
1779
|
+
return await s.tap(target, hold_ms=hold_ms)
|
|
1780
|
+
self._refuse_disabled(found) # even `force` doesn't touch what the app has switched off
|
|
1781
|
+
if a.get("force"):
|
|
1782
|
+
done = Noted(await s.tap(target, hold_ms=hold_ms)) # at its centre, whatever is drawn there
|
|
1783
|
+
done.note = self._forced(view, found)
|
|
1784
|
+
return done
|
|
1785
|
+
self._guard(view, found)
|
|
1786
|
+
if (point := self._touch_point(found)) is None:
|
|
1787
|
+
return await s.tap(target, hold_ms=hold_ms)
|
|
1788
|
+
await s.tap(ui.Target(x=point[0], y=point[1]), hold_ms=hold_ms)
|
|
1789
|
+
return f"{verb} {found.line(actions=False)} at its visible part ({round(point[0])}, {round(point[1])})."
|
|
1790
|
+
|
|
1791
|
+
def _forced(self, view, element: ui.Element) -> str:
|
|
1792
|
+
"""The note for a touch made with `force` at the centre of an element that the checks would have refused,
|
|
1793
|
+
or whose centre a bar is drawn over (there the unforced touch goes to the part that shows): with a real
|
|
1794
|
+
touch, `force` doesn't make the hidden element act. Empty when nothing was in the way."""
|
|
1795
|
+
reason = ""
|
|
1796
|
+
try:
|
|
1797
|
+
if not element.in_alert:
|
|
1798
|
+
self._refuse_behind_alert(view)
|
|
1799
|
+
except Refused as e:
|
|
1800
|
+
reason = str(e).split(":", 1)[0]
|
|
1801
|
+
try:
|
|
1802
|
+
if not reason and self._touch_point(element) is not None:
|
|
1803
|
+
centre = element.center
|
|
1804
|
+
over = [b for b in element.covered_by if b.rect[0] <= centre[0] < b.rect[0] + b.rect[2]
|
|
1805
|
+
and b.rect[1] <= centre[1] < b.rect[1] + b.rect[3]]
|
|
1806
|
+
reason = "under_tab_bar" if over and over[0].kind == "TabBar" else "covered"
|
|
1807
|
+
except Refused as e:
|
|
1808
|
+
reason = str(e).split(":", 1)[0]
|
|
1809
|
+
return NOTE_FORCED.format(reason=reason) if reason else ""
|
|
1810
|
+
|
|
1811
|
+
async def _tap(self, view, a: dict) -> str:
|
|
1812
|
+
return await self._touch(view, a, TAP_MS)
|
|
1813
|
+
|
|
1814
|
+
async def _long_press(self, view, a: dict) -> str:
|
|
1815
|
+
return await self._touch(view, a, int(float(a.get("seconds", 1.0)) * 1000))
|
|
1816
|
+
|
|
1817
|
+
async def _toggle(self, view, a: dict) -> str:
|
|
1818
|
+
target = _target(a)
|
|
1819
|
+
if target.point is not None:
|
|
1820
|
+
x, y = target.point
|
|
1821
|
+
found = ui.at_point(view.items, target.point, "switch")
|
|
1822
|
+
if found is None:
|
|
1823
|
+
raise Refused("not_actionable", f"no_action: Nothing at ({round(x)}, {round(y)}) takes a toggle.")
|
|
1824
|
+
else:
|
|
1825
|
+
found = self._switch_first(view.items, target)
|
|
1826
|
+
line = found.line(actions=False)
|
|
1827
|
+
if found.role != "switch" or found.checked is None:
|
|
1828
|
+
raise Refused("not_actionable", f"no_action: {line} is not a switch")
|
|
1829
|
+
self._refuse_disabled(found)
|
|
1830
|
+
forced, point, shown = "", None, None
|
|
1831
|
+
if a.get("force"):
|
|
1832
|
+
forced = self._forced(view, found)
|
|
1833
|
+
else:
|
|
1834
|
+
self._guard(view, found)
|
|
1835
|
+
point = shown = self._touch_point(found)
|
|
1836
|
+
if found.width > WIDE_SWITCH * found.height:
|
|
1837
|
+
point = self._switch_control(view.items, found, shown)
|
|
1838
|
+
want = a.get("value")
|
|
1839
|
+
if want is not None and found.checked == want:
|
|
1840
|
+
return Unchanged(f"{line} is already {'true' if want else 'false'}.")
|
|
1841
|
+
await self.session.tap(ui.Target(ref=found.ref) if point is None else ui.Target(x=point[0], y=point[1]))
|
|
1842
|
+
# Until the tree is read, claim no new state: the first line is the verdict's to write
|
|
1843
|
+
where = "" if shown is None else f" at its visible part ({round(point[0])}, {round(point[1])})"
|
|
1844
|
+
done = Toggled(f"Tapped {dataclasses.replace(found, checked=None, value='').line(actions=False)}{where}.")
|
|
1845
|
+
done.ref, done.before, done.want, done.flipped = found.ref, found, want, not found.checked
|
|
1846
|
+
done.note = forced
|
|
1847
|
+
return done
|
|
1848
|
+
|
|
1849
|
+
def _switch_control(self, items: list[ui.Element], switch: ui.Element,
|
|
1850
|
+
shown: tuple[float, float] | None) -> tuple[float, float] | None:
|
|
1851
|
+
"""Where the control of a switch that is a whole row sits (iOS Settings: "Allow Notifications" is one
|
|
1852
|
+
element 350x29 at x 40; a tap on its middle lands on the label and toggles nothing, seen on the phone
|
|
1853
|
+
2026-11-11). The control is at the end away from the label: the right in a left-to-right layout, the
|
|
1854
|
+
left in Arabic or Hebrew. The row's label shows as its own element inside it; without one, the right.
|
|
1855
|
+
Partly covered (`shown`, the visible part's middle when the centre is covered): the control's end at that
|
|
1856
|
+
height. A bar over the control itself is refused `covered`, as tap refuses: a finger couldn't reach it."""
|
|
1857
|
+
x, y, w, h = switch.rect
|
|
1858
|
+
label = next((e for e in items if e is not switch and e.label and e.label == switch.label
|
|
1859
|
+
and x <= e.center[0] <= x + w and y <= e.center[1] <= y + h), None)
|
|
1860
|
+
leading_right = label is not None and label.center[0] > x + w / 2
|
|
1861
|
+
point = (x + SWITCH_FROM_RIGHT if leading_right else x + w - SWITCH_FROM_RIGHT,
|
|
1862
|
+
y + h / 2 if shown is None else shown[1])
|
|
1863
|
+
over = next((b for b in switch.covered_by if ui._holds(b.rect, point)), None)
|
|
1864
|
+
if over is not None:
|
|
1865
|
+
hint = SCROLL_HINT if over.kind in ui.COVERING_BARS else "move or close it first."
|
|
1866
|
+
raise Refused("not_actionable", f"covered: {switch.line(actions=False)} is covered by {over.line}; {hint}")
|
|
1867
|
+
return point
|
|
1868
|
+
|
|
1869
|
+
@staticmethod
|
|
1870
|
+
def _switch_first(items: list[ui.Element], target: ui.Target) -> ui.Element:
|
|
1871
|
+
"""The element a target names, preferring a switch: a row and its switch often share a label."""
|
|
1872
|
+
if target.ref is None:
|
|
1873
|
+
try:
|
|
1874
|
+
return ui.resolve([e for e in items if e.role == "switch"], target)
|
|
1875
|
+
except Refused as e:
|
|
1876
|
+
if e.code != "not_found":
|
|
1877
|
+
raise # several switches match: the agent picks one
|
|
1878
|
+
return ui.resolve(items, target)
|
|
1879
|
+
|
|
1880
|
+
async def _back(self, view, a: dict) -> str:
|
|
1881
|
+
"""Close an open popover by a touch outside it (AGENT-TOOLS v1.4); else tap the navigation bar's back
|
|
1882
|
+
button. A bar without one is a root screen (or has only Cancel,
|
|
1883
|
+
Done and the like): nothing is tapped. The edge swipe is only for screens with no bar at all."""
|
|
1884
|
+
s = self.session
|
|
1885
|
+
back = await self._still_back()
|
|
1886
|
+
if back.dismiss is not None: # AGENT-TOOLS v1.4: what is open on top closes first, as "up one screen"
|
|
1887
|
+
await s.tap(ui.Target(x=back.dismiss[0], y=back.dismiss[1]))
|
|
1888
|
+
return "Closed the sheet."
|
|
1889
|
+
behind = getattr(s, "behind", [])
|
|
1890
|
+
# A sheet in front with no region of its own to tap. A back button it hides belongs to the screen behind
|
|
1891
|
+
# (it can't be reached, and where the backdrop lets a touch through it would leave that screen with the
|
|
1892
|
+
# sheet open); one that shows is the sheet's own and is tapped as usual.
|
|
1893
|
+
if behind and not (back.point is not None and not any(ui._holds(e.rect, back.point) for e, _ in behind)):
|
|
1894
|
+
return await self._close_sheet(view)
|
|
1895
|
+
if back.point is not None:
|
|
1896
|
+
await s.tap(ui.Target(x=back.point[0], y=back.point[1]))
|
|
1897
|
+
return "Tapped the back button."
|
|
1898
|
+
if back.bar:
|
|
1899
|
+
raise Refused("not_actionable", "no_action: the navigation bar has no back button")
|
|
1900
|
+
width, height = s.screen
|
|
1901
|
+
await s.swipe(2, height / 2, 0.6 * width, height / 2, 300)
|
|
1902
|
+
return "Swiped from the left edge."
|
|
1903
|
+
|
|
1904
|
+
async def _still_back(self) -> ui.Back:
|
|
1905
|
+
"""Where `back` would touch, read until two reads in a row agree. A sheet that is still sliding in has
|
|
1906
|
+
more room above it than it will have: the middle of that room, tapped half a second later, is inside the
|
|
1907
|
+
sheet. On the phone that touch landed on an app in the share sheet and opened it (2026-11-10, over Wi-Fi,
|
|
1908
|
+
where a read and the touch after it are further apart; Mail came to the front). So nothing is touched
|
|
1909
|
+
until the sheet stands still; a sheet that never does is `no_action`."""
|
|
1910
|
+
s = self.session
|
|
1911
|
+
|
|
1912
|
+
def where(back: ui.Back) -> tuple:
|
|
1913
|
+
behind = getattr(s, "behind", [])
|
|
1914
|
+
front = behind[0][1] if behind else None
|
|
1915
|
+
box = ui.sheet_box(front, getattr(s, "front_items", None) or [], s.screen) if behind else None
|
|
1916
|
+
return back.dismiss, back.point, box
|
|
1917
|
+
back = await s.back_point()
|
|
1918
|
+
seen = where(back)
|
|
1919
|
+
if back.dismiss is None and not getattr(s, "behind", []):
|
|
1920
|
+
return back # nothing is open on top: a bar's back button doesn't slide
|
|
1921
|
+
for _ in range(BACK_STILL_TRIES):
|
|
1922
|
+
await asyncio.sleep(BACK_STILL_PAUSE)
|
|
1923
|
+
back = await s.back_point()
|
|
1924
|
+
now = where(back)
|
|
1925
|
+
if now == seen:
|
|
1926
|
+
return back
|
|
1927
|
+
seen = now
|
|
1928
|
+
raise Refused("not_actionable", "no_action: what is open on top is still moving; call back again in a moment")
|
|
1929
|
+
|
|
1930
|
+
async def _close_sheet(self, view) -> str:
|
|
1931
|
+
"""Close the sheet in front the two ways a person tries from outside it: a touch beside it, on what it
|
|
1932
|
+
dims, then pulling it down from its top edge. After each, the screen is read: the sheet is closed when the
|
|
1933
|
+
layer in front is gone or is another one. A sheet that takes neither is `no_action`, naming it."""
|
|
1934
|
+
s = self.session
|
|
1935
|
+
|
|
1936
|
+
def front() -> str | None:
|
|
1937
|
+
behind = getattr(s, "behind", [])
|
|
1938
|
+
if not behind:
|
|
1939
|
+
return None
|
|
1940
|
+
return behind[0][1].line(ref=False, actions=False) if behind[0][1] is not None else ""
|
|
1941
|
+
was = front()
|
|
1942
|
+
box = ui.sheet_box(getattr(s, "behind")[0][1], getattr(s, "front_items", None) or view.items, s.screen)
|
|
1943
|
+
if box is not None:
|
|
1944
|
+
top, bottom = s.last_insets
|
|
1945
|
+
beside = ui.beside_sheet(box, s.screen, SCROLL_TOP_INSET if top is None else top,
|
|
1946
|
+
s.screen[1] - (SCROLL_BOTTOM_INSET if bottom is None else bottom))
|
|
1947
|
+
if beside is not None:
|
|
1948
|
+
await s.tap(ui.Target(x=beside[0], y=beside[1]))
|
|
1949
|
+
await self._silent_read(view)
|
|
1950
|
+
if front() != was:
|
|
1951
|
+
return "Closed the sheet."
|
|
1952
|
+
if (pull := ui.sheet_pull(box, s.screen)) is not None:
|
|
1953
|
+
await s.swipe(*pull, 300)
|
|
1954
|
+
await self._silent_read(view)
|
|
1955
|
+
if front() != was:
|
|
1956
|
+
return "Closed the sheet."
|
|
1957
|
+
raise Refused("not_actionable", "no_action: nothing changed after back" + self._still_open())
|
|
1958
|
+
|
|
1959
|
+
async def _settle_pending(self) -> None:
|
|
1960
|
+
"""XCTest no longer waits for the app to go idle, so after an action that wasn't settled (`look: false`)
|
|
1961
|
+
nothing may act or read until the screen is still (cap RESOLVE_SETTLE)."""
|
|
1962
|
+
if self.unsettled:
|
|
1963
|
+
await self._settle(self._mark()[0], cap=RESOLVE_SETTLE)
|
|
1964
|
+
self.unsettled = False
|
|
1965
|
+
|
|
1966
|
+
async def _silent_read(self, view) -> bool:
|
|
1967
|
+
"""Read the live tree for target resolution, once the screen is still (cap RESOLVE_SETTLE; with no
|
|
1968
|
+
frames that is a `settle_fallback` wait, 0.6 s). Read mid-redraw, an element can be briefly missing.
|
|
1969
|
+
True when a re-read could show something else: the stream moved, ended capped, or has no frames."""
|
|
1970
|
+
s = self.session
|
|
1971
|
+
done = await self._settle(self._mark()[0], cap=RESOLVE_SETTLE)
|
|
1972
|
+
self.unsettled = False
|
|
1973
|
+
mark = self._mark() # taken before the read: a frame that changes meanwhile counts against the list
|
|
1974
|
+
tree = await s.refresh()
|
|
1975
|
+
view.alert_now = _alert_key(tree.alert) # the alert as this read shows it, not as the last answer did
|
|
1976
|
+
view.read_keyboard = tree.keyboard
|
|
1977
|
+
known = {e.ref for e in view.items} # what came back may get its old ref: look in both lists
|
|
1978
|
+
held = sorted([*view.items, *(e for e in view.shown if e.ref not in known)], key=lambda e: (e.y, e.x))
|
|
1979
|
+
# no action of the agent's came before this read: the content moved only if the elements say so
|
|
1980
|
+
view.items = changes.carry_refs(held, s.items, view.top_ref, moves_content=None)
|
|
1981
|
+
view.top_ref = max(view.top_ref, changes.highest(view.items))
|
|
1982
|
+
view.read_at, view.read_mark = self.control.clock(), mark # the next target in the same window needn't read again
|
|
1983
|
+
s.items = view.items
|
|
1984
|
+
return done.capped or mark[1] is None or not alike(self._mark()[1], mark[1])
|
|
1985
|
+
|
|
1986
|
+
@staticmethod
|
|
1987
|
+
def _missing(items: list[ui.Element], target: ui.Target) -> bool:
|
|
1988
|
+
"""Does the target resolve to `not_found` (not `ambiguous`) on this list?"""
|
|
1989
|
+
try:
|
|
1990
|
+
ui.resolve(items, target)
|
|
1991
|
+
except Refused as e:
|
|
1992
|
+
return e.code == "not_found"
|
|
1993
|
+
return False
|
|
1994
|
+
|
|
1995
|
+
async def settle(self, after_seq: int, still: float | None = None, cap: float = CAP,
|
|
1996
|
+
since: Frame | None = None, early: int | None = None) -> tuple[float, bool]:
|
|
1997
|
+
"""(seconds waited, changed): see _settle."""
|
|
1998
|
+
done = await self._settle(after_seq, still, cap, since, early)
|
|
1999
|
+
return done.seconds, done.changed
|
|
2000
|
+
|
|
2001
|
+
async def _settle(self, after_seq: int, still: float | None = None, cap: float = CAP,
|
|
2002
|
+
since: Frame | None = None, early: int | None = None) -> Settled:
|
|
2003
|
+
"""Wait for the screen to be still, as the stream shows it. `after_seq` is the newest frame when the
|
|
2004
|
+
phone call returned: the settle ends only on a later frame, since earlier ones may predate the effect.
|
|
2005
|
+
WebDriverAgent streams continuously even when nothing changes, so a silence is a stall, not stillness.
|
|
2006
|
+
|
|
2007
|
+
`early` is the newest frame when the gesture went out. A tap's call returns about 200 ms after its effect
|
|
2008
|
+
shows, so the frames in between count too, but only once one of them is unlike `since`: they can then
|
|
2009
|
+
already hold the still run, and the settle ends at the first frame after the return that continues it.
|
|
2010
|
+
|
|
2011
|
+
Still: at least 2 frames alike (stream.alike) spanning `still` seconds (at least 2.5 median frame gaps,
|
|
2012
|
+
so a slow stream isn't called still too soon). Each frame is compared with the first of its run, so a
|
|
2013
|
+
slow fade, close to the last frame every time, is not still. `changed`: a frame is unlike the one at the
|
|
2014
|
+
start of the action (`since`, from _mark; None means there was no frame, so any frame counts).
|
|
2015
|
+
`changed` is False only when the answer is certain: 3 frames, all like `since`, and `quiet` seconds.
|
|
2016
|
+
A stall, no frames at all (it sleeps `settle_fallback` in total, first-frame wait included), too
|
|
2017
|
+
few frames, or the cap all report changed=True, so the tree gets read."""
|
|
2018
|
+
gap = self.hub.gap()
|
|
2019
|
+
still = max(self.settle_still if still is None else still, 2.5 * gap if gap else 0.0)
|
|
2020
|
+
stall = max(3 * gap if gap else 0.0, still)
|
|
2021
|
+
loop = asyncio.get_running_loop()
|
|
2022
|
+
began = loop.time()
|
|
2023
|
+
deadline = began + cap
|
|
2024
|
+
seq, differs, run, run_start, first_of_run = after_seq, False, 0, began, None
|
|
2025
|
+
if early is not None:
|
|
2026
|
+
last = None
|
|
2027
|
+
for arrived, frame in self.hub.since(early, upto=after_seq):
|
|
2028
|
+
differs = differs or not alike(frame, since)
|
|
2029
|
+
if alike(frame, first_of_run):
|
|
2030
|
+
run += 1
|
|
2031
|
+
else:
|
|
2032
|
+
run, run_start, first_of_run = 1, arrived, frame
|
|
2033
|
+
last = arrived
|
|
2034
|
+
if not differs: # nothing showed during the call: only frames after the return count
|
|
2035
|
+
run, run_start, first_of_run = 0, began, None
|
|
2036
|
+
elif (gap and last is not None and run >= 2 and began - run_start >= still
|
|
2037
|
+
and began - last <= 1.5 * gap):
|
|
2038
|
+
return Settled(0.0, True, False) # changed, then still up to a fresh frame at the return
|
|
2039
|
+
while (left := deadline - loop.time()) > 0:
|
|
2040
|
+
first = seq == after_seq
|
|
2041
|
+
wait = min(left, self.settle_first_frame if first else stall)
|
|
2042
|
+
frame = await self.hub.next_after(seq, timeout=wait)
|
|
2043
|
+
now = loop.time()
|
|
2044
|
+
if frame is None:
|
|
2045
|
+
if not first and wait == left:
|
|
2046
|
+
break # the cap came before the stall did
|
|
2047
|
+
if first: # no frames at all: the stream is down or still opening
|
|
2048
|
+
await asyncio.sleep(max(0.0, self.settle_fallback - (now - began)))
|
|
2049
|
+
return Settled(loop.time() - began, True, False) # or a stall
|
|
2050
|
+
seq = frame.seq
|
|
2051
|
+
differs = differs or not alike(frame, since)
|
|
2052
|
+
if alike(frame, first_of_run):
|
|
2053
|
+
run += 1
|
|
2054
|
+
else:
|
|
2055
|
+
run, run_start, first_of_run = 1, now, frame
|
|
2056
|
+
if run >= 2 and now - run_start >= still:
|
|
2057
|
+
if differs or (seq - after_seq >= 3 and now - began >= self.settle_quiet):
|
|
2058
|
+
return Settled(now - began, differs, False)
|
|
2059
|
+
return Settled(loop.time() - began, True, differs)
|
|
2060
|
+
|
|
2061
|
+
def _mark_read(self, view) -> None:
|
|
2062
|
+
"""A tree read starts: note the stream's picture (its SHA-1 and, if it's recent, its bytes), so a
|
|
2063
|
+
screenshot shows the moment the list is of, not a later one."""
|
|
2064
|
+
view.read_mark = self._mark()
|
|
2065
|
+
frame = self.hub.latest
|
|
2066
|
+
fresh = frame is not None and self.hub.now_ms() - frame.at <= self.shot_fresh * 1000
|
|
2067
|
+
view.read_frame = frame if fresh else None
|
|
2068
|
+
|
|
2069
|
+
def _mark(self) -> tuple[int, Frame | None]:
|
|
2070
|
+
"""(seq, the newest frame): where the stream stood at this moment."""
|
|
2071
|
+
frame = self.hub.latest
|
|
2072
|
+
return (frame.seq, frame) if frame else (0, None)
|
|
2073
|
+
|
|
2074
|
+
def _shot_args(self, args: dict, *, default: bool, custom: bool) -> tuple[int, bool] | None:
|
|
2075
|
+
"""(width, labels) when a screenshot is wanted, else None. Only `look` takes width and labels;
|
|
2076
|
+
an action's screenshot uses look's defaults (AGENT-TOOLS §3.5)."""
|
|
2077
|
+
if not custom:
|
|
2078
|
+
return (shotlib.DEFAULT_WIDTH, False) if args.get("screenshot", default) else None
|
|
2079
|
+
width = args.get("width", shotlib.DEFAULT_WIDTH)
|
|
2080
|
+
if isinstance(width, bool) or not isinstance(width, (int, float)) or width != int(width):
|
|
2081
|
+
raise Refused("bad_params", "width must be a whole number of pixels")
|
|
2082
|
+
if not shotlib.MIN_WIDTH <= width <= shotlib.MAX_WIDTH:
|
|
2083
|
+
raise Refused("bad_params", f"width must be {shotlib.MIN_WIDTH}-{shotlib.MAX_WIDTH} pixels, not {width:g}")
|
|
2084
|
+
if not args.get("screenshot", default):
|
|
2085
|
+
return None
|
|
2086
|
+
return int(width), bool(args.get("labels", False))
|
|
2087
|
+
|
|
2088
|
+
async def _newest_frame(self) -> Frame | None:
|
|
2089
|
+
"""The newest stream frame if it's recent, else the next one if it comes soon, else None."""
|
|
2090
|
+
latest = self.hub.latest
|
|
2091
|
+
if latest is not None and self.hub.now_ms() - latest.at <= self.shot_fresh * 1000:
|
|
2092
|
+
return latest
|
|
2093
|
+
return await self.hub.next_after(latest.seq if latest else 0, timeout=self.shot_wait)
|
|
2094
|
+
|
|
2095
|
+
async def _add_shot(self, answer: dict, view, shot: tuple[int, bool] | None, at_read: bool = True) -> dict:
|
|
2096
|
+
"""Add the picture to an answer: from the stream (full resolution), else the phone's /screenshot.
|
|
2097
|
+
A picture that can't be had is a Note line, never an error. Decoding and resizing happen only here,
|
|
2098
|
+
so only when a picture was asked for. The picture is the frame from when the tree was read (`at_read`),
|
|
2099
|
+
else the newest one (the no-change shortcut read no tree); /screenshot only when there is no frame."""
|
|
2100
|
+
if shot is None:
|
|
2101
|
+
return answer
|
|
2102
|
+
width, labels = shot
|
|
2103
|
+
try:
|
|
2104
|
+
frame = view.read_frame if at_read else await self._newest_frame()
|
|
2105
|
+
try:
|
|
2106
|
+
source = await jpeg_of(frame) if frame is not None else None # a video picture is encoded here
|
|
2107
|
+
except PictureGone: # it was one of the older pictures, whose pixels are let go
|
|
2108
|
+
source = None
|
|
2109
|
+
turn = getattr(self.session, "turn", 0)
|
|
2110
|
+
if source is not None and turn and getattr(frame, "picture", None) is None:
|
|
2111
|
+
source = await self.turned(frame.seq, source, turn) # landscape: the stream's frame comes as captured
|
|
2112
|
+
if source is None:
|
|
2113
|
+
source = await self._read(self.session.screenshot_png)
|
|
2114
|
+
boxes = [(e.ref, *e.rect) for e in view.items if e.actions] if labels else None
|
|
2115
|
+
jpeg, size, skipped = await asyncio.to_thread(shotlib.render, source, self.session.screen, width, boxes)
|
|
2116
|
+
except Exception as e:
|
|
2117
|
+
answer["text"] += f"\nNote: no screenshot: {str(e) or type(e).__name__}"
|
|
2118
|
+
return answer
|
|
2119
|
+
if skipped:
|
|
2120
|
+
answer["text"] += "\nNote: labels skipped: the screen orientation changed."
|
|
2121
|
+
answer["image"], answer["image_size"] = base64.b64encode(jpeg).decode(), [size[0], size[1]]
|
|
2122
|
+
return answer
|
|
2123
|
+
|
|
2124
|
+
async def _look(self, view, shot: tuple[int, bool] | None, lines: list, notes: list) -> dict:
|
|
2125
|
+
async def read():
|
|
2126
|
+
self._mark_read(view)
|
|
2127
|
+
text, image = await self.session.look(screenshot=False)
|
|
2128
|
+
view.items = view.shown = self.session.items
|
|
2129
|
+
view.shown_off = self._off_screen_now()
|
|
2130
|
+
view.top_ref, view.app, view.seen = changes.highest(view.items), self.session.app, self.control.generation
|
|
2131
|
+
view.alert, view.keyboard = _alert_key(self.session.last_alert), self.session.last_keyboard
|
|
2132
|
+
view.read_keyboard = view.keyboard
|
|
2133
|
+
view.alert_now = view.alert
|
|
2134
|
+
view.read_at, view.moved, view.dropped = self.control.clock(), False, False
|
|
2135
|
+
view.unlooked = 0
|
|
2136
|
+
return text, image
|
|
2137
|
+
try:
|
|
2138
|
+
text, image = await self._read(read)
|
|
2139
|
+
except Exception as e:
|
|
2140
|
+
if not lines:
|
|
2141
|
+
raise
|
|
2142
|
+
view.moved = True # no list came back: the screen has probably changed
|
|
2143
|
+
return _answer(lines, [*notes, f"couldn't look afterwards: {e}."])
|
|
2144
|
+
return await self._add_shot(_answer([*lines, text], notes, image), view, shot)
|
|
2145
|
+
|
|
2146
|
+
async def _after(self, view, result: str, notes: list, full: bool, moves_content: bool | None = False,
|
|
2147
|
+
seconds: float | None = None, snap: Snapshot | None = None, drifted: bool = False) -> dict:
|
|
2148
|
+
"""An action's answer: its result line, then what changed on screen since the agent's last answer
|
|
2149
|
+
(`view.shown`), or the full list when the app changed or `full`, then `Settled in` (left out when
|
|
2150
|
+
`seconds` is None: `wait` did nothing to settle), then the notes. Refs follow their elements."""
|
|
2151
|
+
s = self.session
|
|
2152
|
+
tail = [] if seconds is None else [settled_line(seconds)]
|
|
2153
|
+
app_before, listed_before = view.app, view.shown
|
|
2154
|
+
|
|
2155
|
+
async def read():
|
|
2156
|
+
self._mark_read(view)
|
|
2157
|
+
read_now = snap or await s.snapshot() # `snap`: a read the caller already made (wait_for's last poll)
|
|
2158
|
+
s.items = read_now.tree.items
|
|
2159
|
+
app_changed = read_now.app is not None and view.app is not None and read_now.app != view.app
|
|
2160
|
+
head = _furniture(view, read_now.tree.alert, read_now.tree.keyboard)
|
|
2161
|
+
up = changes.update(view.items, read_now.tree.items, app_changed, view.top_ref, view.shown, full, head,
|
|
2162
|
+
moves_content)
|
|
2163
|
+
return read_now, up
|
|
2164
|
+
try:
|
|
2165
|
+
snap, up = await self._read(read)
|
|
2166
|
+
except Exception as e:
|
|
2167
|
+
view.moved = True # no list came back: the screen has probably changed
|
|
2168
|
+
return _answer([result, *tail], [*notes, f"couldn't look afterwards: {e}."])
|
|
2169
|
+
covered = not snap.tree.items and bool(view.shown) # the agent saw elements; now there are none
|
|
2170
|
+
s.items = view.items = view.shown = up.items
|
|
2171
|
+
view.top_ref, view.app = up.top, snap.app or view.app
|
|
2172
|
+
view.alert, view.keyboard = _alert_key(snap.tree.alert), snap.tree.keyboard
|
|
2173
|
+
view.read_keyboard = view.keyboard
|
|
2174
|
+
view.alert_now = view.alert
|
|
2175
|
+
view.seen, view.read_at, view.moved, view.dropped = (self.control.generation, self.control.clock(),
|
|
2176
|
+
False, False)
|
|
2177
|
+
# what changed off screen (v1.6), against what lay there at the agent's last answer
|
|
2178
|
+
off_was, view.shown_off = view.shown_off, self._off_screen_of(snap.tree)
|
|
2179
|
+
# ...unless the old list is no baseline: another app is in front now, or the full list was forced (a
|
|
2180
|
+
# person acted, the line dropped, the agent had never looked). A place only says "the same element" when
|
|
2181
|
+
# nothing can have moved the content since the agent's last answer.
|
|
2182
|
+
same_app = not (snap.app is not None and app_before is not None and snap.app != app_before)
|
|
2183
|
+
still = not (moves_content is not False or drifted or view.unlooked)
|
|
2184
|
+
off = (changes.off_screen(off_was, view.shown_off, by_place=still, listed=listed_before)
|
|
2185
|
+
if same_app and not full else [])
|
|
2186
|
+
if off and not up.full:
|
|
2187
|
+
up.lines = [line for line in up.lines if line != changes.NO_CHANGE] + off
|
|
2188
|
+
earlier, view.unlooked = view.unlooked, 0
|
|
2189
|
+
if earlier and not up.full:
|
|
2190
|
+
# the block is measured from the agent's last list, so it holds those actions' changes too: a
|
|
2191
|
+
# first-time agent closed a sheet it had opened with look: false and read "No visible change."
|
|
2192
|
+
notes = [*notes, NOTE_SPAN.format(n=earlier + 1)]
|
|
2193
|
+
if up.full:
|
|
2194
|
+
tree = snap.tree
|
|
2195
|
+
head = ui.describe(up.items, s.screen, snap.scale, snap.app, tree.alert, tree.keyboard, tree.more,
|
|
2196
|
+
(tree.above, tree.below))
|
|
2197
|
+
# the full list because most of the screen changed: what changed off screen still follows it, under
|
|
2198
|
+
# a heading of its own (bare `~` lines after a list read as a second, broken block)
|
|
2199
|
+
beyond = [changes.OFF_SCREEN, *off] if off else []
|
|
2200
|
+
return _answer([result, head, *beyond, *tail], [*notes, *([NOTE_COVERED] if covered else [])])
|
|
2201
|
+
return _answer([result, *up.lines, *tail], notes)
|
|
2202
|
+
|
|
2203
|
+
def _off_screen_now(self) -> list[ui.Element]:
|
|
2204
|
+
"""What the session's last read found off screen, without what an open sheet hides (as in the header's
|
|
2205
|
+
counts). Only right inside the read itself: another client's read may come next."""
|
|
2206
|
+
s = self.session
|
|
2207
|
+
hidden = {id(e) for e, _ in getattr(s, "behind", [])}
|
|
2208
|
+
return [e for e in getattr(s, "offscreen", []) if id(e) not in hidden]
|
|
2209
|
+
|
|
2210
|
+
def _off_screen_of(self, tree: ui.Tree) -> list[ui.Element]:
|
|
2211
|
+
"""The same from one read's own tree. (A tree a test made by hand has no off-screen list of its own: the
|
|
2212
|
+
session's stands in.)"""
|
|
2213
|
+
if not tree.offscreen and not tree.behind:
|
|
2214
|
+
return self._off_screen_now()
|
|
2215
|
+
hidden = {id(e) for e, _ in tree.behind}
|
|
2216
|
+
return [e for e in tree.offscreen if id(e) not in hidden]
|
|
2217
|
+
|
|
2218
|
+
def _current(self, view) -> bool:
|
|
2219
|
+
"""Is the agent's last list still the screen? Not if a person or another agent acted since
|
|
2220
|
+
(a different generation), if an action returned no list since, or if it is older than
|
|
2221
|
+
LIVE_AGE seconds and the screen may have changed meanwhile."""
|
|
2222
|
+
if view.seen != self.control.generation or view.moved or view.dropped:
|
|
2223
|
+
return False
|
|
2224
|
+
return self.control.clock() - view.read_at < LIVE_AGE or self._screen_unchanged(view)
|
|
2225
|
+
|
|
2226
|
+
def _screen_unchanged(self, view) -> bool:
|
|
2227
|
+
"""Does the stream show the picture the list was read against? Only a fresh newest frame that
|
|
2228
|
+
matches the one at read time says so; with no stream the answer is no, and the list is read again."""
|
|
2229
|
+
frame, (_, was) = self.hub.latest, view.read_mark
|
|
2230
|
+
if frame is None or was is None or self.hub.now_ms() - frame.at > FRESH_FRAME * 1000:
|
|
2231
|
+
return False
|
|
2232
|
+
return alike(frame, was)
|
|
2233
|
+
|
|
2234
|
+
async def _read(self, fn):
|
|
2235
|
+
"""A read through the queue; after a dropped line, once more when reconnected."""
|
|
2236
|
+
for attempt in (1, 2):
|
|
2237
|
+
try:
|
|
2238
|
+
return await self.control.read(lambda: self._connected_only(fn))
|
|
2239
|
+
except ConnectionLost as e:
|
|
2240
|
+
self.lost(e)
|
|
2241
|
+
if attempt == 2:
|
|
2242
|
+
raise Refused("reconnecting", "lost the connection to the phone; try again") from None
|
|
2243
|
+
await self.wait_connected()
|
|
2244
|
+
|
|
2245
|
+
# ---- a person, from the live view ----
|
|
2246
|
+
async def viewer_message(self, message: dict) -> dict | None:
|
|
2247
|
+
"""One message from a live view (docs/VIEWER-PROTOCOL.md). Returns an error message for
|
|
2248
|
+
that viewer, or None. Unknown types are ignored."""
|
|
2249
|
+
kind = message.get("type")
|
|
2250
|
+
try:
|
|
2251
|
+
if kind == "pause":
|
|
2252
|
+
self.control.set_paused(bool(message.get("on")))
|
|
2253
|
+
self.broadcast(self.status())
|
|
2254
|
+
return None
|
|
2255
|
+
if kind == "quality":
|
|
2256
|
+
await self.set_quality(str(message.get("preset")))
|
|
2257
|
+
return None
|
|
2258
|
+
if kind not in ("gesture", "scroll", "text", "key", "button"):
|
|
2259
|
+
return None # "overlay" comes in part 2; anything else is unknown
|
|
2260
|
+
if self.state != "live":
|
|
2261
|
+
return {"type": "error", "code": {"locked": "locked", "not-answering": "not_answering"}.get(self.state, "reconnecting"),
|
|
2262
|
+
"message": f"not connected to the phone ({self.detail or self.state})"}
|
|
2263
|
+
run, action, point = self._person_action(kind, message)
|
|
2264
|
+
self._person_at = time.monotonic()
|
|
2265
|
+
|
|
2266
|
+
async def quickened():
|
|
2267
|
+
await self._quicken() # so the picture after the input comes at full rate (V1)
|
|
2268
|
+
return await run()
|
|
2269
|
+
await self.control.person(lambda: self._connected_only(quickened))
|
|
2270
|
+
self._announce("person", action, point=self._point(point))
|
|
2271
|
+
return None
|
|
2272
|
+
except Refused as e:
|
|
2273
|
+
return {"type": "error", "code": e.code, "message": str(e)}
|
|
2274
|
+
except ConnectionLost as e:
|
|
2275
|
+
self.lost(e)
|
|
2276
|
+
return {"type": "error", "code": "reconnecting",
|
|
2277
|
+
"message": "the connection dropped; your input may not have reached the phone"}
|
|
2278
|
+
except (ValueError, KeyError, TypeError, IndexError) as e:
|
|
2279
|
+
return {"type": "error", "code": "unsupported", "message": f"bad {kind} message: {e}"}
|
|
2280
|
+
except Exception as e:
|
|
2281
|
+
return {"type": "error", "message": str(e)[:300]}
|
|
2282
|
+
|
|
2283
|
+
def _person_action(self, kind: str, m: dict):
|
|
2284
|
+
"""(coroutine function to run, kind for the action message, start point in points)."""
|
|
2285
|
+
s = self.session
|
|
2286
|
+
if kind == "gesture":
|
|
2287
|
+
g = classify(m["points"], s.screen)
|
|
2288
|
+
if g.kind == "drag":
|
|
2289
|
+
return (lambda: s.path(list(g.points))), "drag", g.points[0][1:]
|
|
2290
|
+
_, x, y = g.points[0]
|
|
2291
|
+
return (lambda: s.tap(ui.Target(x=x, y=y), hold_ms=g.hold_ms)), g.kind, (x, y)
|
|
2292
|
+
if kind == "scroll":
|
|
2293
|
+
x0, y0, x1, y1 = ui.scroll_swipe(float(m["x"]), float(m["y"]), float(m.get("dx", 0)),
|
|
2294
|
+
float(m.get("dy", 0)), s.screen)
|
|
2295
|
+
return (lambda: s.scroll(x0, y0, x1, y1)), "scroll", (x0, y0) # a wheel stops where it should
|
|
2296
|
+
if kind == "text":
|
|
2297
|
+
text = str(m["text"])
|
|
2298
|
+
if not text:
|
|
2299
|
+
raise ValueError("no text")
|
|
2300
|
+
return (lambda: s.type_text(text)), "text", None
|
|
2301
|
+
if kind == "key":
|
|
2302
|
+
name = str(m["key"])
|
|
2303
|
+
if name not in ("backspace", "enter"):
|
|
2304
|
+
raise ValueError(f"unknown key {name!r}")
|
|
2305
|
+
return (lambda: s.key(name)), name, None
|
|
2306
|
+
name = str(m["name"])
|
|
2307
|
+
if name not in BUTTONS:
|
|
2308
|
+
raise ValueError(f"unknown button {name!r}")
|
|
2309
|
+
return (lambda: s.press_button(name)), name, None
|
|
2310
|
+
|
|
2311
|
+
async def set_quality(self, preset: str) -> None:
|
|
2312
|
+
if preset not in PRESETS:
|
|
2313
|
+
raise ValueError(f"preset must be one of {', '.join(PRESETS)}")
|
|
2314
|
+
if self.state in ("live", "locked"):
|
|
2315
|
+
await self._read(lambda: self.session.settings({**PRESETS[preset], **NO_IDLE_WAIT}))
|
|
2316
|
+
self._fps = int(PRESETS[preset]["mjpegServerFramerate"]) # full rate; the pace slows it again when still
|
|
2317
|
+
self.preset = preset
|
|
2318
|
+
if self.video is not None:
|
|
2319
|
+
self.video.set_quality(preset)
|
|
2320
|
+
self.broadcast(self.status())
|
|
2321
|
+
|
|
2322
|
+
# ---- the stream and shutting down ----
|
|
2323
|
+
async def _open_stream(self):
|
|
2324
|
+
try:
|
|
2325
|
+
await asyncio.wait_for(self._connected.wait(), 30)
|
|
2326
|
+
except asyncio.TimeoutError:
|
|
2327
|
+
raise ConnectionLost("not connected to the phone yet") from None
|
|
2328
|
+
return await self.session.device.open_stream()
|
|
2329
|
+
|
|
2330
|
+
async def start_video(self, tries: int = 3, gap: float = 3.0) -> str:
|
|
2331
|
+
"""Set up the H.264 live view: the hub, the AirPlay receiver, and Screen Mirroring on the phone, started
|
|
2332
|
+
the way the owner would (mirror.py). The app that was in front comes back afterwards. Returns what
|
|
2333
|
+
happened in words; it never raises, since the JPEG stream stays as the picture."""
|
|
2334
|
+
from . import airplay, mirror, video
|
|
2335
|
+
s = self.session
|
|
2336
|
+
if not video.available():
|
|
2337
|
+
return self._video_note("PyAV isn't installed (pip install tapwright[video])")
|
|
2338
|
+
uxplay = airplay.find()
|
|
2339
|
+
if uxplay is None:
|
|
2340
|
+
return self._video_note("no AirPlay receiver found (install UxPlay, or set TAPWRIGHT_UXPLAY)")
|
|
2341
|
+
try:
|
|
2342
|
+
scale = s.scale or 3.0
|
|
2343
|
+
pixels = (round(s.screen[0] * scale), round(s.screen[1] * scale))
|
|
2344
|
+
receiver = airplay.Receiver(uxplay, 0, pixels, self.log)
|
|
2345
|
+
hub = video.VideoHub(port=0, screen_width=lambda: s.screen[0], mirroring=lambda: receiver.mirroring,
|
|
2346
|
+
log=self.log)
|
|
2347
|
+
receiver.args = airplay.command(uxplay, hub.port, pixels)
|
|
2348
|
+
receiver.start()
|
|
2349
|
+
self.video, self._receiver = hub, receiver
|
|
2350
|
+
if self.settle_on_video:
|
|
2351
|
+
self._start_feeder()
|
|
2352
|
+
hub.set_quality(self.preset)
|
|
2353
|
+
said = ""
|
|
2354
|
+
for attempt in range(tries):
|
|
2355
|
+
await asyncio.sleep(gap) # the receiver takes a moment to show up in the phone's list
|
|
2356
|
+
async with self.control.exclusive():
|
|
2357
|
+
self._require_connected()
|
|
2358
|
+
front = await s.foreground_bundle_id()
|
|
2359
|
+
try:
|
|
2360
|
+
said = await mirror.start(s, airplay.NAME)
|
|
2361
|
+
except mirror.MirrorError as e:
|
|
2362
|
+
said = str(e)
|
|
2363
|
+
continue
|
|
2364
|
+
finally:
|
|
2365
|
+
if front and front != "com.apple.springboard":
|
|
2366
|
+
await s.launch(front) # back to the app that was in front
|
|
2367
|
+
break
|
|
2368
|
+
for view in self.control.clients.values():
|
|
2369
|
+
view.dropped = True # the phone left the app for a moment: every agent's list is old
|
|
2370
|
+
self.broadcast(self.status())
|
|
2371
|
+
return self._video_note(said, ok=receiver.mirroring or said.startswith(("Mirroring", "Already")))
|
|
2372
|
+
except Exception as e: # the JPEG stream stays: video is an extra
|
|
2373
|
+
return self._video_note(f"couldn't start: {e}")
|
|
2374
|
+
|
|
2375
|
+
def _start_feeder(self) -> None:
|
|
2376
|
+
"""While `video` is live its pictures feed `hub` (and the MJPEG stream stays closed); when it isn't, the
|
|
2377
|
+
MJPEG stream does, as before. The feeder checks which, so mirroring may stop and start again."""
|
|
2378
|
+
if self._feeder_task is None and self.video is not None:
|
|
2379
|
+
self._feeder = PictureFeeder(self.video, self.hub, log=self.log)
|
|
2380
|
+
self._feeder_task = asyncio.create_task(self._feeder.run(), name="feeder")
|
|
2381
|
+
|
|
2382
|
+
def _video_note(self, text: str, ok: bool = False) -> str:
|
|
2383
|
+
self.log(f"video: {text}" if ok else f"video: {text}; the live view stays on the JPEG stream")
|
|
2384
|
+
return text
|
|
2385
|
+
|
|
2386
|
+
async def close(self) -> None:
|
|
2387
|
+
if self._feeder_task is not None:
|
|
2388
|
+
self._feeder_task.cancel()
|
|
2389
|
+
await asyncio.gather(self._feeder_task, return_exceptions=True)
|
|
2390
|
+
await self.pump.stop()
|
|
2391
|
+
self.hub.close()
|
|
2392
|
+
if self._video_task is not None:
|
|
2393
|
+
self._video_task.cancel()
|
|
2394
|
+
if self._receiver is not None:
|
|
2395
|
+
await asyncio.to_thread(self._receiver.stop)
|
|
2396
|
+
if self.video is not None:
|
|
2397
|
+
await asyncio.to_thread(self.video.close) # joins its threads
|
|
2398
|
+
try:
|
|
2399
|
+
await asyncio.wait_for(self.session.close(), 10)
|
|
2400
|
+
except Exception as e:
|
|
2401
|
+
self.log(f"closing: {e}")
|