tapwright 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tapwright/__init__.py +2 -0
- tapwright/__main__.py +5 -0
- tapwright/airplay.py +103 -0
- tapwright/changes.py +264 -0
- tapwright/cli.py +374 -0
- tapwright/client.py +151 -0
- tapwright/control.py +189 -0
- tapwright/daemon.py +2401 -0
- tapwright/descriptions.py +186 -0
- tapwright/device.py +495 -0
- tapwright/errors.py +13 -0
- tapwright/mcp_server.py +266 -0
- tapwright/mirror.py +91 -0
- tapwright/rtp.py +109 -0
- tapwright/server.py +435 -0
- tapwright/session.py +672 -0
- tapwright/shot.py +42 -0
- tapwright/signing.py +201 -0
- tapwright/skill/SKILL.md +203 -0
- tapwright/skill/agents/openai.yaml +4 -0
- tapwright/skill/references/setup.md +102 -0
- tapwright/skills.py +60 -0
- tapwright/store.py +130 -0
- tapwright/stream.py +405 -0
- tapwright/tapline.py +118 -0
- tapwright/ui.py +1242 -0
- tapwright/video.py +486 -0
- tapwright/viewer.html +2060 -0
- tapwright-1.0.0.dist-info/METADATA +254 -0
- tapwright-1.0.0.dist-info/RECORD +34 -0
- tapwright-1.0.0.dist-info/WHEEL +5 -0
- tapwright-1.0.0.dist-info/entry_points.txt +2 -0
- tapwright-1.0.0.dist-info/licenses/LICENSE +21 -0
- tapwright-1.0.0.dist-info/top_level.txt +1 -0
tapwright/mcp_server.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""tapwright as an MCP server (stdio): tools an agent uses to test apps on your iPhone.
|
|
2
|
+
|
|
3
|
+
The tools, targets and error codes are described in descriptions.py (and docs/AGENT-TOOLS.md).
|
|
4
|
+
|
|
5
|
+
The phone belongs to `tapwright serve`, which this server starts when needed; a person
|
|
6
|
+
may be watching or driving in its live view at the same time.
|
|
7
|
+
"""
|
|
8
|
+
import asyncio
|
|
9
|
+
import base64
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import secrets
|
|
14
|
+
import uuid
|
|
15
|
+
from datetime import datetime, timezone
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Literal
|
|
18
|
+
|
|
19
|
+
from mcp.server.mcpserver import MCPServer
|
|
20
|
+
from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError
|
|
21
|
+
from mcp.types import CallToolResult, ImageContent, TextContent, ToolAnnotations
|
|
22
|
+
from pydantic import BaseModel, ConfigDict, ValidationError
|
|
23
|
+
|
|
24
|
+
from . import client, device, ui
|
|
25
|
+
from .descriptions import INSTRUCTIONS, describe
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger("tapwright")
|
|
28
|
+
|
|
29
|
+
SHOTS_DIR: Path | None = None # set by `tapwright mcp --screenshots DIR`
|
|
30
|
+
KEEP_SHOTS = 50
|
|
31
|
+
_SHOT_NAME = re.compile(r"^\d{8}-\d{6}-\d{3}-[0-9a-f]{4}-[a-z_]+\.jpg$")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def check_screenshots_dir(directory: Path) -> None:
|
|
35
|
+
"""Fail at startup, with a clear message, when `--screenshots DIR` can't be written to."""
|
|
36
|
+
try:
|
|
37
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
probe = directory / f".tapwright-{secrets.token_hex(4)}.tmp"
|
|
39
|
+
probe.write_bytes(b"")
|
|
40
|
+
probe.unlink()
|
|
41
|
+
except OSError as e:
|
|
42
|
+
raise SystemExit(f"tapwright mcp: can't write screenshots to --screenshots {directory}: {e.strerror or e}") from e
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def save_screenshot(directory: Path, tool: str, jpeg: bytes, size, now: datetime | None = None) -> str:
|
|
46
|
+
"""Write the picture to DIR/<UTC time>-<tool>.jpg (a short random part keeps two in one millisecond apart),
|
|
47
|
+
keep the newest 50 of ours, and return the answer line. Written under a temporary name first, so
|
|
48
|
+
nothing ever reads half a picture."""
|
|
49
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
50
|
+
now = now or datetime.now(timezone.utc)
|
|
51
|
+
stem = f"{now:%Y%m%d-%H%M%S}-{now.microsecond // 1000:03d}-{secrets.token_hex(2)}-{tool}"
|
|
52
|
+
path, temporary = directory / f"{stem}.jpg", directory / f".{stem}.tmp"
|
|
53
|
+
temporary.write_bytes(jpeg)
|
|
54
|
+
os.replace(temporary, path)
|
|
55
|
+
ours = sorted(p for p in directory.iterdir() if _SHOT_NAME.match(p.name)) # UTC names sort by time
|
|
56
|
+
for old in ours[:-KEEP_SHOTS]:
|
|
57
|
+
try:
|
|
58
|
+
old.unlink()
|
|
59
|
+
except OSError:
|
|
60
|
+
pass
|
|
61
|
+
return f"Screenshot: {path} ({size[0]}x{size[1]})"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _result(text: str, image: str | None = None, is_error: bool = False) -> CallToolResult:
|
|
65
|
+
"""One text item, plus the screenshot (base64 JPEG) when there is one."""
|
|
66
|
+
content: list[TextContent | ImageContent] = [TextContent(type="text", text=text)]
|
|
67
|
+
if image:
|
|
68
|
+
content.append(ImageContent(type="image", data=image, mime_type="image/jpeg"))
|
|
69
|
+
return CallToolResult(content=content, is_error=is_error)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _error(code: str, message: str) -> CallToolResult:
|
|
73
|
+
"""A failed call: `Error <code>: <message>` as the only content, with is_error set."""
|
|
74
|
+
return _result(f"Error {code}: {message}", is_error=True)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class TapwrightServer(MCPServer):
|
|
78
|
+
"""MCPServer whose own failures (arguments the schema rejects, a tool that raised) use the same
|
|
79
|
+
`Error <code>: <message>` form as the daemon's refusals."""
|
|
80
|
+
|
|
81
|
+
async def call_tool(self, name, arguments, context=None):
|
|
82
|
+
try:
|
|
83
|
+
return await super().call_tool(name, arguments, context)
|
|
84
|
+
except UnexpectedToolError:
|
|
85
|
+
logger.exception("tool %s crashed", name)
|
|
86
|
+
return _error("internal", f"{name} failed unexpectedly. This is a bug; report it.")
|
|
87
|
+
except ToolError as e:
|
|
88
|
+
if isinstance(e.__cause__, ValidationError): # the field names and what's wrong, not the input
|
|
89
|
+
errors = e.__cause__.errors()
|
|
90
|
+
lines = [f"{'.'.join(str(part) for part in err['loc'])}: {err['msg'].splitlines()[0]}"
|
|
91
|
+
for err in errors]
|
|
92
|
+
if any(err["loc"][:1] == ("target",) for err in errors):
|
|
93
|
+
lines.append(ui.TARGET_HELP)
|
|
94
|
+
return _error("bad_params", "; ".join(lines))
|
|
95
|
+
if str(e).startswith("Unknown tool"):
|
|
96
|
+
tools = ", ".join(t.name for t in self._tool_manager.list_tools())
|
|
97
|
+
return _error("unsupported", f"no tool named {name!r}; the tools are {tools}")
|
|
98
|
+
return _error("bad_params", str(e).removeprefix(f"Error executing tool {name}: "))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _unset_means_absent(schema: dict) -> None:
|
|
102
|
+
"""Drop the null branch from every field, so a client must leave an unset key out. Otherwise a `null`
|
|
103
|
+
counts as present in the `oneOf` below. The server still accepts nulls and strips them."""
|
|
104
|
+
schema["oneOf"] = [{"required": ["ref"]}, {"required": ["id"]}, {"required": ["label"]},
|
|
105
|
+
{"required": ["text"]}, {"required": ["x", "y"]}]
|
|
106
|
+
for prop in schema["properties"].values():
|
|
107
|
+
prop.pop("default", None)
|
|
108
|
+
kinds = [k for k in prop.pop("anyOf", []) if k.get("type") != "null"]
|
|
109
|
+
if kinds:
|
|
110
|
+
prop.update(kinds[0])
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class Target(BaseModel):
|
|
114
|
+
"""What to act on: exactly one of ref, id, label, text, or x with y. The daemon checks the rule and
|
|
115
|
+
answers `bad_params` with an example when it isn't met."""
|
|
116
|
+
model_config = ConfigDict(extra="forbid", json_schema_extra=_unset_means_absent)
|
|
117
|
+
ref: str | None = None
|
|
118
|
+
id: str | None = None
|
|
119
|
+
label: str | None = None
|
|
120
|
+
text: str | None = None
|
|
121
|
+
x: float | None = None
|
|
122
|
+
y: float | None = None
|
|
123
|
+
index: int | None = None
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
READ_ONLY = ToolAnnotations(read_only_hint=True)
|
|
127
|
+
server = TapwrightServer("tapwright", instructions=INSTRUCTIONS)
|
|
128
|
+
CLIENT_ID = uuid.uuid4().hex # this agent's own last look, kept by the daemon
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
async def _tool(tool: str, /, **args) -> CallToolResult:
|
|
132
|
+
"""Run one tool in the daemon; its answer is one text item, plus the screenshot if there is one."""
|
|
133
|
+
args = {k: (v.model_dump(exclude_none=True) if isinstance(v, BaseModel) else v) for k, v in args.items()}
|
|
134
|
+
try:
|
|
135
|
+
answer = await asyncio.to_thread(client.call, tool, args, CLIENT_ID)
|
|
136
|
+
except client.DaemonError as e:
|
|
137
|
+
return _error(e.code, str(e))
|
|
138
|
+
except Exception: # the daemon unreachable: never crash the server
|
|
139
|
+
logger.exception("tool %s crashed", tool)
|
|
140
|
+
return _error("internal", f"{tool} failed unexpectedly. This is a bug; report it.")
|
|
141
|
+
text, image = answer.get("text") or "(no answer)", answer.get("image")
|
|
142
|
+
if image and SHOTS_DIR is not None: # file mode: one text item, the picture on disk
|
|
143
|
+
try:
|
|
144
|
+
text += "\n" + save_screenshot(SHOTS_DIR, tool, base64.b64decode(image), answer.get("image_size") or [0, 0])
|
|
145
|
+
except OSError as e: # the user chose file mode: no inline picture instead, but the answer stays
|
|
146
|
+
text += f"\nNote: no screenshot: couldn't save to {SHOTS_DIR}: {e.strerror or e}"
|
|
147
|
+
image = None
|
|
148
|
+
return _result(text, image)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@server.tool(annotations=READ_ONLY, description=describe("look"), structured_output=False)
|
|
152
|
+
async def look(screenshot: bool = True, width: int = 600, labels: bool = False) -> CallToolResult:
|
|
153
|
+
return await _tool("look", screenshot=screenshot, width=width, labels=labels)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@server.tool(description=describe("tap"), structured_output=False)
|
|
157
|
+
async def tap(target: Target, force: bool = False, look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
158
|
+
return await _tool("tap", target=target, force=force, look=look, screenshot=screenshot)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@server.tool(description=describe("long_press"), structured_output=False)
|
|
162
|
+
async def long_press(target: Target, seconds: float = 1.0, force: bool = False, look: bool = True,
|
|
163
|
+
screenshot: bool = False) -> CallToolResult:
|
|
164
|
+
return await _tool("long_press", target=target, seconds=seconds, force=force, look=look, screenshot=screenshot)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@server.tool(description=describe("type_text"), structured_output=False)
|
|
168
|
+
async def type_text(text: str, target: Target | None = None, replace: bool = True, submit: bool = False,
|
|
169
|
+
force: bool = False, look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
170
|
+
return await _tool("type_text", text=text, target=target, replace=replace, submit=submit, force=force,
|
|
171
|
+
look=look, screenshot=screenshot)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@server.tool(description=describe("scroll"), structured_output=False)
|
|
175
|
+
async def scroll(direction: Literal["up", "down", "left", "right"] | None = None, amount: float = 0.8,
|
|
176
|
+
to: Literal["start", "end", "target"] | None = None, target: Target | None = None, look: bool = True,
|
|
177
|
+
screenshot: bool = False) -> CallToolResult:
|
|
178
|
+
return await _tool("scroll", direction=direction, amount=amount, to=to, target=target, look=look,
|
|
179
|
+
screenshot=screenshot)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@server.tool(description=describe("toggle"), structured_output=False)
|
|
183
|
+
async def toggle(target: Target, value: bool | None = None, force: bool = False, look: bool = True,
|
|
184
|
+
screenshot: bool = False) -> CallToolResult:
|
|
185
|
+
return await _tool("toggle", target=target, value=value, force=force, look=look, screenshot=screenshot)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@server.tool(description=describe("back"), structured_output=False)
|
|
189
|
+
async def back(look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
190
|
+
return await _tool("back", look=look, screenshot=screenshot)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@server.tool(description=describe("open_url"), structured_output=False)
|
|
194
|
+
async def open_url(url: str, look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
195
|
+
return await _tool("open_url", url=url, look=look, screenshot=screenshot)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
@server.tool(annotations=READ_ONLY, description=describe("wait"), structured_output=False)
|
|
199
|
+
async def wait(seconds: float, look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
200
|
+
return await _tool("wait", seconds=seconds, look=look, screenshot=screenshot)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
@server.tool(annotations=READ_ONLY, description=describe("wait_for"), structured_output=False)
|
|
204
|
+
async def wait_for(text: str | None = None, id: str | None = None, label: str | None = None, gone: bool = False,
|
|
205
|
+
timeout: float = 10.0, look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
206
|
+
return await _tool("wait_for", text=text, id=id, label=label, gone=gone, timeout=timeout, look=look,
|
|
207
|
+
screenshot=screenshot)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@server.tool(annotations=READ_ONLY, description=describe("apps"), structured_output=False)
|
|
211
|
+
async def apps() -> CallToolResult:
|
|
212
|
+
return await _tool("apps")
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@server.tool(description=describe("open_app"), structured_output=False)
|
|
216
|
+
async def open_app(app: str, look: bool = True, screenshot: bool = False) -> CallToolResult:
|
|
217
|
+
return await _tool("open_app", app=app, look=look, screenshot=screenshot)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@server.tool(annotations=READ_ONLY, description=describe("viewer"), structured_output=False)
|
|
221
|
+
async def viewer() -> CallToolResult:
|
|
222
|
+
# Unlike the other tools this one may start the daemon (find_or_start), since only this side knows its port and key.
|
|
223
|
+
try:
|
|
224
|
+
info = await asyncio.to_thread(client.find_or_start)
|
|
225
|
+
except client.DaemonError as e:
|
|
226
|
+
return _error(e.code, str(e))
|
|
227
|
+
return _result(f"Live view: {client.viewer_url(info)}\n"
|
|
228
|
+
"The link carries a key and is for the user only: give it to no one else, and don't repeat it.")
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@server.tool(description=describe("close_app"), structured_output=False)
|
|
232
|
+
async def close_app(app: str) -> CallToolResult:
|
|
233
|
+
return await _tool("close_app", app=app)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@server.tool(description=describe("alert"), structured_output=False)
|
|
237
|
+
async def alert(action: Literal["accept", "dismiss", "text"], look: bool = True,
|
|
238
|
+
screenshot: bool = False) -> CallToolResult:
|
|
239
|
+
return await _tool("alert", action=action, look=look, screenshot=screenshot)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@server.tool(description=describe("press_button"), structured_output=False)
|
|
243
|
+
async def press_button(name: Literal["home", "volumeUp", "volumeDown"], look: bool = True,
|
|
244
|
+
screenshot: bool = False) -> CallToolResult:
|
|
245
|
+
return await _tool("press_button", name=name, look=look, screenshot=screenshot)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@server.tool(description=describe("swipe"), structured_output=False)
|
|
249
|
+
async def swipe(x0: float, y0: float, x1: float, y1: float, seconds: float = 0.35, look: bool = True,
|
|
250
|
+
screenshot: bool = False) -> CallToolResult:
|
|
251
|
+
return await _tool("swipe", x0=x0, y0=y0, x1=x1, y1=y1, seconds=seconds, look=look, screenshot=screenshot)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
@server.tool(description=describe("rotate"), structured_output=False)
|
|
255
|
+
async def rotate(to: Literal["portrait", "landscape_left", "landscape_right"], look: bool = True,
|
|
256
|
+
screenshot: bool = False) -> CallToolResult:
|
|
257
|
+
return await _tool("rotate", to=to, look=look, screenshot=screenshot)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def run(screenshots: str | None = None, apps: list[str] | None = None, dedicated: bool = False) -> None:
|
|
261
|
+
client.set_scope(apps, dedicated)
|
|
262
|
+
global SHOTS_DIR
|
|
263
|
+
SHOTS_DIR = Path(screenshots).expanduser().resolve() if screenshots else None
|
|
264
|
+
if SHOTS_DIR is not None:
|
|
265
|
+
check_screenshots_dir(SHOTS_DIR)
|
|
266
|
+
device.run(server.run_stdio_async)
|
tapwright/mirror.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Starting and stopping the phone's Screen Mirroring to this computer, the way the owner would: Control Center,
|
|
2
|
+
Screen Mirroring, the receiver's name. The phone's own AirPlay mirror is the best picture Tapwright can get (the
|
|
3
|
+
phone's H.264 at full size, 60 frames a second, a frame only when the screen changes), and iOS offers no other way
|
|
4
|
+
to start it than these touches.
|
|
5
|
+
|
|
6
|
+
Each step reads the screen first and stops when it doesn't show what the step expects, so a touch never goes to
|
|
7
|
+
something unknown. Measured on an iPhone 14 Pro Max, iOS 26.5 (2026-11-10)."""
|
|
8
|
+
import asyncio
|
|
9
|
+
|
|
10
|
+
from . import ui
|
|
11
|
+
from .control import Refused
|
|
12
|
+
|
|
13
|
+
OPEN_FROM = (30.0, 4.0) # where the swipe that opens Control Center starts: this far from the right edge, from the top
|
|
14
|
+
OPEN_BY = 420.0 # points the finger travels down
|
|
15
|
+
STEP = 1.2 # seconds Control Center and its sheets take to settle
|
|
16
|
+
CONNECT = 3.0 # seconds the phone takes to start mirroring after the receiver is picked
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MirrorError(Refused):
|
|
20
|
+
"""A step didn't find what it needed; nothing was touched from that step on."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, message: str):
|
|
23
|
+
super().__init__("unsupported", message)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def control(items: list[ui.Element]) -> ui.Element | None:
|
|
27
|
+
"""Control Center's Screen Mirroring control: the button if listed, else the image that carries the name."""
|
|
28
|
+
named = [e for e in items if e.label.strip().lower() == "screen mirroring"]
|
|
29
|
+
return next((e for e in named if "tap" in e.actions), named[0] if named else None)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def receiver(items: list[ui.Element], name: str) -> ui.Element | None:
|
|
33
|
+
"""The row of the receiver called `name` in the Screen Mirroring list (exact, ignoring case)."""
|
|
34
|
+
return next((e for e in items if e.label.strip().lower() == name.lower() and "tap" in e.actions), None)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def stop_button(items: list[ui.Element]) -> ui.Element | None:
|
|
38
|
+
return next((e for e in items if e.label.strip().lower() == "stop mirroring" and "tap" in e.actions), None)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def mirroring_to(items: list[ui.Element], name: str) -> bool:
|
|
42
|
+
"""Does the Screen Mirroring list show `name` as the one in use?"""
|
|
43
|
+
row = receiver(items, name)
|
|
44
|
+
return row is not None and (row.selected or row.value == "1")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def _open_list(session) -> list[ui.Element]:
|
|
48
|
+
"""Open Control Center and its Screen Mirroring list; the list's elements."""
|
|
49
|
+
width = session.screen[0]
|
|
50
|
+
x = width - OPEN_FROM[0]
|
|
51
|
+
await session.swipe(x, OPEN_FROM[1], x, OPEN_FROM[1] + OPEN_BY, 300)
|
|
52
|
+
await asyncio.sleep(STEP)
|
|
53
|
+
centre = (await session.refresh()).items
|
|
54
|
+
button = control(centre)
|
|
55
|
+
if button is None:
|
|
56
|
+
await session.press_button("home")
|
|
57
|
+
raise MirrorError("Control Center didn't open, or it has no Screen Mirroring control; nothing was changed.")
|
|
58
|
+
await session.tap(ui.Target(ref=button.ref))
|
|
59
|
+
await asyncio.sleep(STEP)
|
|
60
|
+
return (await session.refresh()).items
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def start(session, name: str) -> str:
|
|
64
|
+
"""Start mirroring to the receiver called `name`. Leaves the phone on the Home screen: the caller brings its
|
|
65
|
+
app back. Raises MirrorError when the receiver isn't offered (not running, or another network)."""
|
|
66
|
+
rows = await _open_list(session)
|
|
67
|
+
if mirroring_to(rows, name):
|
|
68
|
+
await session.press_button("home")
|
|
69
|
+
return f"Already mirroring to {name}."
|
|
70
|
+
row = receiver(rows, name)
|
|
71
|
+
if row is None:
|
|
72
|
+
await session.press_button("home")
|
|
73
|
+
raise MirrorError(f'the phone doesn\'t offer "{name}" under Screen Mirroring. The phone and this computer '
|
|
74
|
+
"must be on the same Wi-Fi, and the receiver must be running.")
|
|
75
|
+
await session.tap(ui.Target(ref=row.ref))
|
|
76
|
+
await asyncio.sleep(CONNECT)
|
|
77
|
+
await session.press_button("home")
|
|
78
|
+
return f"Mirroring to {name}."
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def stop(session) -> str:
|
|
82
|
+
"""Stop mirroring, if the phone is mirroring. Leaves the phone on the Home screen."""
|
|
83
|
+
rows = await _open_list(session)
|
|
84
|
+
button = stop_button(rows)
|
|
85
|
+
if button is None:
|
|
86
|
+
await session.press_button("home")
|
|
87
|
+
return "The phone wasn't mirroring."
|
|
88
|
+
await session.tap(ui.Target(ref=button.ref))
|
|
89
|
+
await asyncio.sleep(STEP)
|
|
90
|
+
await session.press_button("home")
|
|
91
|
+
return "Stopped mirroring."
|
tapwright/rtp.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# adapted from phoneuse (MIT), Aleksey Smolenchuk
|
|
2
|
+
"""H.264 over RTP (RFC 6184) back into access units, for the live view's video hub.
|
|
3
|
+
|
|
4
|
+
UxPlay forwards the phone's AirPlay stream as RTP to localhost. This reassembles single NAL unit
|
|
5
|
+
packets, STAP-A aggregates and FU-A fragments into Annex B access units (a start code before each
|
|
6
|
+
NAL unit), one per frame. A packet lost on the way drops the NAL unit it belonged to and is counted;
|
|
7
|
+
the decoder conceals the rest.
|
|
8
|
+
"""
|
|
9
|
+
import struct
|
|
10
|
+
|
|
11
|
+
START = b"\x00\x00\x00\x01"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Depacketizer:
|
|
15
|
+
def __init__(self) -> None:
|
|
16
|
+
self.nals: list[bytes] = [] # NAL units of the access unit being collected
|
|
17
|
+
self.ts: int | None = None # its RTP timestamp
|
|
18
|
+
self.ssrc: int | None = None
|
|
19
|
+
self.seq: int | None = None
|
|
20
|
+
self.fu: bytearray | None = None # the FU-A NAL unit being reassembled
|
|
21
|
+
self.lost = 0 # packets missing (sequence gaps)
|
|
22
|
+
self.restarts = 0 # new streams (SSRC changes: UxPlay or the phone started over)
|
|
23
|
+
|
|
24
|
+
def push(self, pkt: bytes) -> list[bytes]:
|
|
25
|
+
"""Feed one RTP packet; returns the access units it completed (Annex B)."""
|
|
26
|
+
out: list[bytes] = []
|
|
27
|
+
if len(pkt) < 12 or pkt[0] >> 6 != 2:
|
|
28
|
+
return out
|
|
29
|
+
cc, ext, pad, marker = pkt[0] & 0x0F, pkt[0] & 0x10, pkt[0] & 0x20, pkt[1] & 0x80
|
|
30
|
+
seq, ts, ssrc = struct.unpack_from("!HII", pkt, 2)
|
|
31
|
+
off = 12 + 4 * cc
|
|
32
|
+
if ext:
|
|
33
|
+
if len(pkt) < off + 4:
|
|
34
|
+
return out
|
|
35
|
+
off += 4 + 4 * struct.unpack_from("!H", pkt, off + 2)[0]
|
|
36
|
+
end = len(pkt) - (pkt[-1] if pad else 0)
|
|
37
|
+
payload = pkt[off:end]
|
|
38
|
+
if not payload:
|
|
39
|
+
return out
|
|
40
|
+
|
|
41
|
+
if ssrc != self.ssrc:
|
|
42
|
+
if self.ssrc is not None:
|
|
43
|
+
self.restarts += 1
|
|
44
|
+
self.ssrc, self.seq, self.ts, self.nals, self.fu = ssrc, None, None, [], None
|
|
45
|
+
if self.seq is not None:
|
|
46
|
+
gap = (seq - self.seq - 1) & 0xFFFF
|
|
47
|
+
if gap and gap < 0x8000:
|
|
48
|
+
self.lost += gap
|
|
49
|
+
self.fu = None # the fragment in progress is incomplete
|
|
50
|
+
elif gap >= 0x8000:
|
|
51
|
+
return out # a late duplicate or reordered packet: too late for it
|
|
52
|
+
self.seq = seq
|
|
53
|
+
if self.ts is not None and ts != self.ts and self.nals:
|
|
54
|
+
out.append(self._flush()) # the previous frame's marker went missing
|
|
55
|
+
self.ts = ts
|
|
56
|
+
|
|
57
|
+
kind = payload[0] & 0x1F
|
|
58
|
+
if 1 <= kind <= 23:
|
|
59
|
+
self.nals.append(bytes(payload))
|
|
60
|
+
elif kind == 24: # STAP-A: 16-bit size, NAL unit, repeated
|
|
61
|
+
i = 1
|
|
62
|
+
while i + 2 <= len(payload):
|
|
63
|
+
n = struct.unpack_from("!H", payload, i)[0]
|
|
64
|
+
i += 2
|
|
65
|
+
if n == 0 or i + n > len(payload):
|
|
66
|
+
break
|
|
67
|
+
self.nals.append(bytes(payload[i:i + n]))
|
|
68
|
+
i += n
|
|
69
|
+
elif kind == 28 and len(payload) >= 2: # FU-A
|
|
70
|
+
start, stop = payload[1] & 0x80, payload[1] & 0x40
|
|
71
|
+
if start:
|
|
72
|
+
self.fu = bytearray([(payload[0] & 0xE0) | (payload[1] & 0x1F)])
|
|
73
|
+
if self.fu is not None:
|
|
74
|
+
self.fu += payload[2:]
|
|
75
|
+
if stop:
|
|
76
|
+
self.nals.append(bytes(self.fu))
|
|
77
|
+
self.fu = None
|
|
78
|
+
if marker and self.nals:
|
|
79
|
+
out.append(self._flush())
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
def _flush(self) -> bytes:
|
|
83
|
+
au = b"".join(START + n for n in self.nals)
|
|
84
|
+
self.nals = []
|
|
85
|
+
return au
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def split_annexb(data: bytes) -> list[bytes]:
|
|
89
|
+
"""The NAL units (without start codes) of an Annex B byte string."""
|
|
90
|
+
marks: list[int] = []
|
|
91
|
+
i = 0
|
|
92
|
+
while True:
|
|
93
|
+
i = data.find(b"\x00\x00\x01", i)
|
|
94
|
+
if i < 0:
|
|
95
|
+
break
|
|
96
|
+
marks.append(i)
|
|
97
|
+
i += 3
|
|
98
|
+
nals = []
|
|
99
|
+
for n, start in enumerate(marks):
|
|
100
|
+
stop = marks[n + 1] if n + 1 < len(marks) else len(data)
|
|
101
|
+
nal = data[start + 3:stop].rstrip(b"\x00") # the zero of a 4-byte start code belongs to the next one
|
|
102
|
+
if nal:
|
|
103
|
+
nals.append(nal)
|
|
104
|
+
return nals
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def nal_types(au: bytes) -> list[int]:
|
|
108
|
+
"""NAL unit types in an Annex B access unit (for tests and logs)."""
|
|
109
|
+
return [n[0] & 0x1F for n in split_annexb(au)]
|