tapwright 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,266 @@
1
+ """tapwright as an MCP server (stdio): tools an agent uses to test apps on your iPhone.
2
+
3
+ The tools, targets and error codes are described in descriptions.py (and docs/AGENT-TOOLS.md).
4
+
5
+ The phone belongs to `tapwright serve`, which this server starts when needed; a person
6
+ may be watching or driving in its live view at the same time.
7
+ """
8
+ import asyncio
9
+ import base64
10
+ import logging
11
+ import os
12
+ import re
13
+ import secrets
14
+ import uuid
15
+ from datetime import datetime, timezone
16
+ from pathlib import Path
17
+ from typing import Literal
18
+
19
+ from mcp.server.mcpserver import MCPServer
20
+ from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError
21
+ from mcp.types import CallToolResult, ImageContent, TextContent, ToolAnnotations
22
+ from pydantic import BaseModel, ConfigDict, ValidationError
23
+
24
+ from . import client, device, ui
25
+ from .descriptions import INSTRUCTIONS, describe
26
+
27
+ logger = logging.getLogger("tapwright")
28
+
29
+ SHOTS_DIR: Path | None = None # set by `tapwright mcp --screenshots DIR`
30
+ KEEP_SHOTS = 50
31
+ _SHOT_NAME = re.compile(r"^\d{8}-\d{6}-\d{3}-[0-9a-f]{4}-[a-z_]+\.jpg$")
32
+
33
+
34
+ def check_screenshots_dir(directory: Path) -> None:
35
+ """Fail at startup, with a clear message, when `--screenshots DIR` can't be written to."""
36
+ try:
37
+ directory.mkdir(parents=True, exist_ok=True)
38
+ probe = directory / f".tapwright-{secrets.token_hex(4)}.tmp"
39
+ probe.write_bytes(b"")
40
+ probe.unlink()
41
+ except OSError as e:
42
+ raise SystemExit(f"tapwright mcp: can't write screenshots to --screenshots {directory}: {e.strerror or e}") from e
43
+
44
+
45
+ def save_screenshot(directory: Path, tool: str, jpeg: bytes, size, now: datetime | None = None) -> str:
46
+ """Write the picture to DIR/<UTC time>-<tool>.jpg (a short random part keeps two in one millisecond apart),
47
+ keep the newest 50 of ours, and return the answer line. Written under a temporary name first, so
48
+ nothing ever reads half a picture."""
49
+ directory.mkdir(parents=True, exist_ok=True)
50
+ now = now or datetime.now(timezone.utc)
51
+ stem = f"{now:%Y%m%d-%H%M%S}-{now.microsecond // 1000:03d}-{secrets.token_hex(2)}-{tool}"
52
+ path, temporary = directory / f"{stem}.jpg", directory / f".{stem}.tmp"
53
+ temporary.write_bytes(jpeg)
54
+ os.replace(temporary, path)
55
+ ours = sorted(p for p in directory.iterdir() if _SHOT_NAME.match(p.name)) # UTC names sort by time
56
+ for old in ours[:-KEEP_SHOTS]:
57
+ try:
58
+ old.unlink()
59
+ except OSError:
60
+ pass
61
+ return f"Screenshot: {path} ({size[0]}x{size[1]})"
62
+
63
+
64
+ def _result(text: str, image: str | None = None, is_error: bool = False) -> CallToolResult:
65
+ """One text item, plus the screenshot (base64 JPEG) when there is one."""
66
+ content: list[TextContent | ImageContent] = [TextContent(type="text", text=text)]
67
+ if image:
68
+ content.append(ImageContent(type="image", data=image, mime_type="image/jpeg"))
69
+ return CallToolResult(content=content, is_error=is_error)
70
+
71
+
72
+ def _error(code: str, message: str) -> CallToolResult:
73
+ """A failed call: `Error <code>: <message>` as the only content, with is_error set."""
74
+ return _result(f"Error {code}: {message}", is_error=True)
75
+
76
+
77
+ class TapwrightServer(MCPServer):
78
+ """MCPServer whose own failures (arguments the schema rejects, a tool that raised) use the same
79
+ `Error <code>: <message>` form as the daemon's refusals."""
80
+
81
+ async def call_tool(self, name, arguments, context=None):
82
+ try:
83
+ return await super().call_tool(name, arguments, context)
84
+ except UnexpectedToolError:
85
+ logger.exception("tool %s crashed", name)
86
+ return _error("internal", f"{name} failed unexpectedly. This is a bug; report it.")
87
+ except ToolError as e:
88
+ if isinstance(e.__cause__, ValidationError): # the field names and what's wrong, not the input
89
+ errors = e.__cause__.errors()
90
+ lines = [f"{'.'.join(str(part) for part in err['loc'])}: {err['msg'].splitlines()[0]}"
91
+ for err in errors]
92
+ if any(err["loc"][:1] == ("target",) for err in errors):
93
+ lines.append(ui.TARGET_HELP)
94
+ return _error("bad_params", "; ".join(lines))
95
+ if str(e).startswith("Unknown tool"):
96
+ tools = ", ".join(t.name for t in self._tool_manager.list_tools())
97
+ return _error("unsupported", f"no tool named {name!r}; the tools are {tools}")
98
+ return _error("bad_params", str(e).removeprefix(f"Error executing tool {name}: "))
99
+
100
+
101
+ def _unset_means_absent(schema: dict) -> None:
102
+ """Drop the null branch from every field, so a client must leave an unset key out. Otherwise a `null`
103
+ counts as present in the `oneOf` below. The server still accepts nulls and strips them."""
104
+ schema["oneOf"] = [{"required": ["ref"]}, {"required": ["id"]}, {"required": ["label"]},
105
+ {"required": ["text"]}, {"required": ["x", "y"]}]
106
+ for prop in schema["properties"].values():
107
+ prop.pop("default", None)
108
+ kinds = [k for k in prop.pop("anyOf", []) if k.get("type") != "null"]
109
+ if kinds:
110
+ prop.update(kinds[0])
111
+
112
+
113
+ class Target(BaseModel):
114
+ """What to act on: exactly one of ref, id, label, text, or x with y. The daemon checks the rule and
115
+ answers `bad_params` with an example when it isn't met."""
116
+ model_config = ConfigDict(extra="forbid", json_schema_extra=_unset_means_absent)
117
+ ref: str | None = None
118
+ id: str | None = None
119
+ label: str | None = None
120
+ text: str | None = None
121
+ x: float | None = None
122
+ y: float | None = None
123
+ index: int | None = None
124
+
125
+
126
+ READ_ONLY = ToolAnnotations(read_only_hint=True)
127
+ server = TapwrightServer("tapwright", instructions=INSTRUCTIONS)
128
+ CLIENT_ID = uuid.uuid4().hex # this agent's own last look, kept by the daemon
129
+
130
+
131
+ async def _tool(tool: str, /, **args) -> CallToolResult:
132
+ """Run one tool in the daemon; its answer is one text item, plus the screenshot if there is one."""
133
+ args = {k: (v.model_dump(exclude_none=True) if isinstance(v, BaseModel) else v) for k, v in args.items()}
134
+ try:
135
+ answer = await asyncio.to_thread(client.call, tool, args, CLIENT_ID)
136
+ except client.DaemonError as e:
137
+ return _error(e.code, str(e))
138
+ except Exception: # the daemon unreachable: never crash the server
139
+ logger.exception("tool %s crashed", tool)
140
+ return _error("internal", f"{tool} failed unexpectedly. This is a bug; report it.")
141
+ text, image = answer.get("text") or "(no answer)", answer.get("image")
142
+ if image and SHOTS_DIR is not None: # file mode: one text item, the picture on disk
143
+ try:
144
+ text += "\n" + save_screenshot(SHOTS_DIR, tool, base64.b64decode(image), answer.get("image_size") or [0, 0])
145
+ except OSError as e: # the user chose file mode: no inline picture instead, but the answer stays
146
+ text += f"\nNote: no screenshot: couldn't save to {SHOTS_DIR}: {e.strerror or e}"
147
+ image = None
148
+ return _result(text, image)
149
+
150
+
151
+ @server.tool(annotations=READ_ONLY, description=describe("look"), structured_output=False)
152
+ async def look(screenshot: bool = True, width: int = 600, labels: bool = False) -> CallToolResult:
153
+ return await _tool("look", screenshot=screenshot, width=width, labels=labels)
154
+
155
+
156
+ @server.tool(description=describe("tap"), structured_output=False)
157
+ async def tap(target: Target, force: bool = False, look: bool = True, screenshot: bool = False) -> CallToolResult:
158
+ return await _tool("tap", target=target, force=force, look=look, screenshot=screenshot)
159
+
160
+
161
+ @server.tool(description=describe("long_press"), structured_output=False)
162
+ async def long_press(target: Target, seconds: float = 1.0, force: bool = False, look: bool = True,
163
+ screenshot: bool = False) -> CallToolResult:
164
+ return await _tool("long_press", target=target, seconds=seconds, force=force, look=look, screenshot=screenshot)
165
+
166
+
167
+ @server.tool(description=describe("type_text"), structured_output=False)
168
+ async def type_text(text: str, target: Target | None = None, replace: bool = True, submit: bool = False,
169
+ force: bool = False, look: bool = True, screenshot: bool = False) -> CallToolResult:
170
+ return await _tool("type_text", text=text, target=target, replace=replace, submit=submit, force=force,
171
+ look=look, screenshot=screenshot)
172
+
173
+
174
+ @server.tool(description=describe("scroll"), structured_output=False)
175
+ async def scroll(direction: Literal["up", "down", "left", "right"] | None = None, amount: float = 0.8,
176
+ to: Literal["start", "end", "target"] | None = None, target: Target | None = None, look: bool = True,
177
+ screenshot: bool = False) -> CallToolResult:
178
+ return await _tool("scroll", direction=direction, amount=amount, to=to, target=target, look=look,
179
+ screenshot=screenshot)
180
+
181
+
182
+ @server.tool(description=describe("toggle"), structured_output=False)
183
+ async def toggle(target: Target, value: bool | None = None, force: bool = False, look: bool = True,
184
+ screenshot: bool = False) -> CallToolResult:
185
+ return await _tool("toggle", target=target, value=value, force=force, look=look, screenshot=screenshot)
186
+
187
+
188
+ @server.tool(description=describe("back"), structured_output=False)
189
+ async def back(look: bool = True, screenshot: bool = False) -> CallToolResult:
190
+ return await _tool("back", look=look, screenshot=screenshot)
191
+
192
+
193
+ @server.tool(description=describe("open_url"), structured_output=False)
194
+ async def open_url(url: str, look: bool = True, screenshot: bool = False) -> CallToolResult:
195
+ return await _tool("open_url", url=url, look=look, screenshot=screenshot)
196
+
197
+
198
+ @server.tool(annotations=READ_ONLY, description=describe("wait"), structured_output=False)
199
+ async def wait(seconds: float, look: bool = True, screenshot: bool = False) -> CallToolResult:
200
+ return await _tool("wait", seconds=seconds, look=look, screenshot=screenshot)
201
+
202
+
203
+ @server.tool(annotations=READ_ONLY, description=describe("wait_for"), structured_output=False)
204
+ async def wait_for(text: str | None = None, id: str | None = None, label: str | None = None, gone: bool = False,
205
+ timeout: float = 10.0, look: bool = True, screenshot: bool = False) -> CallToolResult:
206
+ return await _tool("wait_for", text=text, id=id, label=label, gone=gone, timeout=timeout, look=look,
207
+ screenshot=screenshot)
208
+
209
+
210
+ @server.tool(annotations=READ_ONLY, description=describe("apps"), structured_output=False)
211
+ async def apps() -> CallToolResult:
212
+ return await _tool("apps")
213
+
214
+
215
+ @server.tool(description=describe("open_app"), structured_output=False)
216
+ async def open_app(app: str, look: bool = True, screenshot: bool = False) -> CallToolResult:
217
+ return await _tool("open_app", app=app, look=look, screenshot=screenshot)
218
+
219
+
220
+ @server.tool(annotations=READ_ONLY, description=describe("viewer"), structured_output=False)
221
+ async def viewer() -> CallToolResult:
222
+ # Unlike the other tools this one may start the daemon (find_or_start), since only this side knows its port and key.
223
+ try:
224
+ info = await asyncio.to_thread(client.find_or_start)
225
+ except client.DaemonError as e:
226
+ return _error(e.code, str(e))
227
+ return _result(f"Live view: {client.viewer_url(info)}\n"
228
+ "The link carries a key and is for the user only: give it to no one else, and don't repeat it.")
229
+
230
+
231
+ @server.tool(description=describe("close_app"), structured_output=False)
232
+ async def close_app(app: str) -> CallToolResult:
233
+ return await _tool("close_app", app=app)
234
+
235
+
236
+ @server.tool(description=describe("alert"), structured_output=False)
237
+ async def alert(action: Literal["accept", "dismiss", "text"], look: bool = True,
238
+ screenshot: bool = False) -> CallToolResult:
239
+ return await _tool("alert", action=action, look=look, screenshot=screenshot)
240
+
241
+
242
+ @server.tool(description=describe("press_button"), structured_output=False)
243
+ async def press_button(name: Literal["home", "volumeUp", "volumeDown"], look: bool = True,
244
+ screenshot: bool = False) -> CallToolResult:
245
+ return await _tool("press_button", name=name, look=look, screenshot=screenshot)
246
+
247
+
248
+ @server.tool(description=describe("swipe"), structured_output=False)
249
+ async def swipe(x0: float, y0: float, x1: float, y1: float, seconds: float = 0.35, look: bool = True,
250
+ screenshot: bool = False) -> CallToolResult:
251
+ return await _tool("swipe", x0=x0, y0=y0, x1=x1, y1=y1, seconds=seconds, look=look, screenshot=screenshot)
252
+
253
+
254
+ @server.tool(description=describe("rotate"), structured_output=False)
255
+ async def rotate(to: Literal["portrait", "landscape_left", "landscape_right"], look: bool = True,
256
+ screenshot: bool = False) -> CallToolResult:
257
+ return await _tool("rotate", to=to, look=look, screenshot=screenshot)
258
+
259
+
260
+ def run(screenshots: str | None = None, apps: list[str] | None = None, dedicated: bool = False) -> None:
261
+ client.set_scope(apps, dedicated)
262
+ global SHOTS_DIR
263
+ SHOTS_DIR = Path(screenshots).expanduser().resolve() if screenshots else None
264
+ if SHOTS_DIR is not None:
265
+ check_screenshots_dir(SHOTS_DIR)
266
+ device.run(server.run_stdio_async)
tapwright/mirror.py ADDED
@@ -0,0 +1,91 @@
1
+ """Starting and stopping the phone's Screen Mirroring to this computer, the way the owner would: Control Center,
2
+ Screen Mirroring, the receiver's name. The phone's own AirPlay mirror is the best picture Tapwright can get (the
3
+ phone's H.264 at full size, 60 frames a second, a frame only when the screen changes), and iOS offers no other way
4
+ to start it than these touches.
5
+
6
+ Each step reads the screen first and stops when it doesn't show what the step expects, so a touch never goes to
7
+ something unknown. Measured on an iPhone 14 Pro Max, iOS 26.5 (2026-11-10)."""
8
+ import asyncio
9
+
10
+ from . import ui
11
+ from .control import Refused
12
+
13
+ OPEN_FROM = (30.0, 4.0) # where the swipe that opens Control Center starts: this far from the right edge, from the top
14
+ OPEN_BY = 420.0 # points the finger travels down
15
+ STEP = 1.2 # seconds Control Center and its sheets take to settle
16
+ CONNECT = 3.0 # seconds the phone takes to start mirroring after the receiver is picked
17
+
18
+
19
+ class MirrorError(Refused):
20
+ """A step didn't find what it needed; nothing was touched from that step on."""
21
+
22
+ def __init__(self, message: str):
23
+ super().__init__("unsupported", message)
24
+
25
+
26
+ def control(items: list[ui.Element]) -> ui.Element | None:
27
+ """Control Center's Screen Mirroring control: the button if listed, else the image that carries the name."""
28
+ named = [e for e in items if e.label.strip().lower() == "screen mirroring"]
29
+ return next((e for e in named if "tap" in e.actions), named[0] if named else None)
30
+
31
+
32
+ def receiver(items: list[ui.Element], name: str) -> ui.Element | None:
33
+ """The row of the receiver called `name` in the Screen Mirroring list (exact, ignoring case)."""
34
+ return next((e for e in items if e.label.strip().lower() == name.lower() and "tap" in e.actions), None)
35
+
36
+
37
+ def stop_button(items: list[ui.Element]) -> ui.Element | None:
38
+ return next((e for e in items if e.label.strip().lower() == "stop mirroring" and "tap" in e.actions), None)
39
+
40
+
41
+ def mirroring_to(items: list[ui.Element], name: str) -> bool:
42
+ """Does the Screen Mirroring list show `name` as the one in use?"""
43
+ row = receiver(items, name)
44
+ return row is not None and (row.selected or row.value == "1")
45
+
46
+
47
+ async def _open_list(session) -> list[ui.Element]:
48
+ """Open Control Center and its Screen Mirroring list; the list's elements."""
49
+ width = session.screen[0]
50
+ x = width - OPEN_FROM[0]
51
+ await session.swipe(x, OPEN_FROM[1], x, OPEN_FROM[1] + OPEN_BY, 300)
52
+ await asyncio.sleep(STEP)
53
+ centre = (await session.refresh()).items
54
+ button = control(centre)
55
+ if button is None:
56
+ await session.press_button("home")
57
+ raise MirrorError("Control Center didn't open, or it has no Screen Mirroring control; nothing was changed.")
58
+ await session.tap(ui.Target(ref=button.ref))
59
+ await asyncio.sleep(STEP)
60
+ return (await session.refresh()).items
61
+
62
+
63
+ async def start(session, name: str) -> str:
64
+ """Start mirroring to the receiver called `name`. Leaves the phone on the Home screen: the caller brings its
65
+ app back. Raises MirrorError when the receiver isn't offered (not running, or another network)."""
66
+ rows = await _open_list(session)
67
+ if mirroring_to(rows, name):
68
+ await session.press_button("home")
69
+ return f"Already mirroring to {name}."
70
+ row = receiver(rows, name)
71
+ if row is None:
72
+ await session.press_button("home")
73
+ raise MirrorError(f'the phone doesn\'t offer "{name}" under Screen Mirroring. The phone and this computer '
74
+ "must be on the same Wi-Fi, and the receiver must be running.")
75
+ await session.tap(ui.Target(ref=row.ref))
76
+ await asyncio.sleep(CONNECT)
77
+ await session.press_button("home")
78
+ return f"Mirroring to {name}."
79
+
80
+
81
+ async def stop(session) -> str:
82
+ """Stop mirroring, if the phone is mirroring. Leaves the phone on the Home screen."""
83
+ rows = await _open_list(session)
84
+ button = stop_button(rows)
85
+ if button is None:
86
+ await session.press_button("home")
87
+ return "The phone wasn't mirroring."
88
+ await session.tap(ui.Target(ref=button.ref))
89
+ await asyncio.sleep(STEP)
90
+ await session.press_button("home")
91
+ return "Stopped mirroring."
tapwright/rtp.py ADDED
@@ -0,0 +1,109 @@
1
+ # adapted from phoneuse (MIT), Aleksey Smolenchuk
2
+ """H.264 over RTP (RFC 6184) back into access units, for the live view's video hub.
3
+
4
+ UxPlay forwards the phone's AirPlay stream as RTP to localhost. This reassembles single NAL unit
5
+ packets, STAP-A aggregates and FU-A fragments into Annex B access units (a start code before each
6
+ NAL unit), one per frame. A packet lost on the way drops the NAL unit it belonged to and is counted;
7
+ the decoder conceals the rest.
8
+ """
9
+ import struct
10
+
11
+ START = b"\x00\x00\x00\x01"
12
+
13
+
14
+ class Depacketizer:
15
+ def __init__(self) -> None:
16
+ self.nals: list[bytes] = [] # NAL units of the access unit being collected
17
+ self.ts: int | None = None # its RTP timestamp
18
+ self.ssrc: int | None = None
19
+ self.seq: int | None = None
20
+ self.fu: bytearray | None = None # the FU-A NAL unit being reassembled
21
+ self.lost = 0 # packets missing (sequence gaps)
22
+ self.restarts = 0 # new streams (SSRC changes: UxPlay or the phone started over)
23
+
24
+ def push(self, pkt: bytes) -> list[bytes]:
25
+ """Feed one RTP packet; returns the access units it completed (Annex B)."""
26
+ out: list[bytes] = []
27
+ if len(pkt) < 12 or pkt[0] >> 6 != 2:
28
+ return out
29
+ cc, ext, pad, marker = pkt[0] & 0x0F, pkt[0] & 0x10, pkt[0] & 0x20, pkt[1] & 0x80
30
+ seq, ts, ssrc = struct.unpack_from("!HII", pkt, 2)
31
+ off = 12 + 4 * cc
32
+ if ext:
33
+ if len(pkt) < off + 4:
34
+ return out
35
+ off += 4 + 4 * struct.unpack_from("!H", pkt, off + 2)[0]
36
+ end = len(pkt) - (pkt[-1] if pad else 0)
37
+ payload = pkt[off:end]
38
+ if not payload:
39
+ return out
40
+
41
+ if ssrc != self.ssrc:
42
+ if self.ssrc is not None:
43
+ self.restarts += 1
44
+ self.ssrc, self.seq, self.ts, self.nals, self.fu = ssrc, None, None, [], None
45
+ if self.seq is not None:
46
+ gap = (seq - self.seq - 1) & 0xFFFF
47
+ if gap and gap < 0x8000:
48
+ self.lost += gap
49
+ self.fu = None # the fragment in progress is incomplete
50
+ elif gap >= 0x8000:
51
+ return out # a late duplicate or reordered packet: too late for it
52
+ self.seq = seq
53
+ if self.ts is not None and ts != self.ts and self.nals:
54
+ out.append(self._flush()) # the previous frame's marker went missing
55
+ self.ts = ts
56
+
57
+ kind = payload[0] & 0x1F
58
+ if 1 <= kind <= 23:
59
+ self.nals.append(bytes(payload))
60
+ elif kind == 24: # STAP-A: 16-bit size, NAL unit, repeated
61
+ i = 1
62
+ while i + 2 <= len(payload):
63
+ n = struct.unpack_from("!H", payload, i)[0]
64
+ i += 2
65
+ if n == 0 or i + n > len(payload):
66
+ break
67
+ self.nals.append(bytes(payload[i:i + n]))
68
+ i += n
69
+ elif kind == 28 and len(payload) >= 2: # FU-A
70
+ start, stop = payload[1] & 0x80, payload[1] & 0x40
71
+ if start:
72
+ self.fu = bytearray([(payload[0] & 0xE0) | (payload[1] & 0x1F)])
73
+ if self.fu is not None:
74
+ self.fu += payload[2:]
75
+ if stop:
76
+ self.nals.append(bytes(self.fu))
77
+ self.fu = None
78
+ if marker and self.nals:
79
+ out.append(self._flush())
80
+ return out
81
+
82
+ def _flush(self) -> bytes:
83
+ au = b"".join(START + n for n in self.nals)
84
+ self.nals = []
85
+ return au
86
+
87
+
88
+ def split_annexb(data: bytes) -> list[bytes]:
89
+ """The NAL units (without start codes) of an Annex B byte string."""
90
+ marks: list[int] = []
91
+ i = 0
92
+ while True:
93
+ i = data.find(b"\x00\x00\x01", i)
94
+ if i < 0:
95
+ break
96
+ marks.append(i)
97
+ i += 3
98
+ nals = []
99
+ for n, start in enumerate(marks):
100
+ stop = marks[n + 1] if n + 1 < len(marks) else len(data)
101
+ nal = data[start + 3:stop].rstrip(b"\x00") # the zero of a 4-byte start code belongs to the next one
102
+ if nal:
103
+ nals.append(nal)
104
+ return nals
105
+
106
+
107
+ def nal_types(au: bytes) -> list[int]:
108
+ """NAL unit types in an Annex B access unit (for tests and logs)."""
109
+ return [n[0] & 0x1F for n in split_annexb(au)]