tapwright 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tapwright-1.0.0/LICENSE +21 -0
- tapwright-1.0.0/PKG-INFO +254 -0
- tapwright-1.0.0/README.md +220 -0
- tapwright-1.0.0/pyproject.toml +49 -0
- tapwright-1.0.0/setup.cfg +4 -0
- tapwright-1.0.0/tapwright/__init__.py +2 -0
- tapwright-1.0.0/tapwright/__main__.py +5 -0
- tapwright-1.0.0/tapwright/airplay.py +103 -0
- tapwright-1.0.0/tapwright/changes.py +264 -0
- tapwright-1.0.0/tapwright/cli.py +374 -0
- tapwright-1.0.0/tapwright/client.py +151 -0
- tapwright-1.0.0/tapwright/control.py +189 -0
- tapwright-1.0.0/tapwright/daemon.py +2401 -0
- tapwright-1.0.0/tapwright/descriptions.py +186 -0
- tapwright-1.0.0/tapwright/device.py +495 -0
- tapwright-1.0.0/tapwright/errors.py +13 -0
- tapwright-1.0.0/tapwright/mcp_server.py +266 -0
- tapwright-1.0.0/tapwright/mirror.py +91 -0
- tapwright-1.0.0/tapwright/rtp.py +109 -0
- tapwright-1.0.0/tapwright/server.py +435 -0
- tapwright-1.0.0/tapwright/session.py +672 -0
- tapwright-1.0.0/tapwright/shot.py +42 -0
- tapwright-1.0.0/tapwright/signing.py +201 -0
- tapwright-1.0.0/tapwright/skill/SKILL.md +203 -0
- tapwright-1.0.0/tapwright/skill/agents/openai.yaml +4 -0
- tapwright-1.0.0/tapwright/skill/references/setup.md +102 -0
- tapwright-1.0.0/tapwright/skills.py +60 -0
- tapwright-1.0.0/tapwright/store.py +130 -0
- tapwright-1.0.0/tapwright/stream.py +405 -0
- tapwright-1.0.0/tapwright/tapline.py +118 -0
- tapwright-1.0.0/tapwright/ui.py +1242 -0
- tapwright-1.0.0/tapwright/video.py +486 -0
- tapwright-1.0.0/tapwright/viewer.html +2060 -0
- tapwright-1.0.0/tapwright.egg-info/PKG-INFO +254 -0
- tapwright-1.0.0/tapwright.egg-info/SOURCES.txt +63 -0
- tapwright-1.0.0/tapwright.egg-info/dependency_links.txt +1 -0
- tapwright-1.0.0/tapwright.egg-info/entry_points.txt +2 -0
- tapwright-1.0.0/tapwright.egg-info/requires.txt +14 -0
- tapwright-1.0.0/tapwright.egg-info/top_level.txt +1 -0
- tapwright-1.0.0/tests/test_airplay.py +41 -0
- tapwright-1.0.0/tests/test_changes.py +451 -0
- tapwright-1.0.0/tests/test_cli.py +24 -0
- tapwright-1.0.0/tests/test_control.py +165 -0
- tapwright-1.0.0/tests/test_daemon.py +5112 -0
- tapwright-1.0.0/tests/test_daemon_video.py +210 -0
- tapwright-1.0.0/tests/test_descriptions.py +92 -0
- tapwright-1.0.0/tests/test_device.py +569 -0
- tapwright-1.0.0/tests/test_mcp.py +180 -0
- tapwright-1.0.0/tests/test_mcp_tools.py +193 -0
- tapwright-1.0.0/tests/test_mirror.py +78 -0
- tapwright-1.0.0/tests/test_offline.py +217 -0
- tapwright-1.0.0/tests/test_rtp.py +100 -0
- tapwright-1.0.0/tests/test_serve.py +253 -0
- tapwright-1.0.0/tests/test_server.py +404 -0
- tapwright-1.0.0/tests/test_session.py +1019 -0
- tapwright-1.0.0/tests/test_shot.py +88 -0
- tapwright-1.0.0/tests/test_skill.py +103 -0
- tapwright-1.0.0/tests/test_stream.py +192 -0
- tapwright-1.0.0/tests/test_stream_pictures.py +303 -0
- tapwright-1.0.0/tests/test_tapline_notice.py +174 -0
- tapwright-1.0.0/tests/test_targets.py +213 -0
- tapwright-1.0.0/tests/test_ui.py +359 -0
- tapwright-1.0.0/tests/test_ui_tools.py +834 -0
- tapwright-1.0.0/tests/test_video.py +284 -0
- tapwright-1.0.0/tests/test_video_pictures.py +50 -0
tapwright-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tapwright contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tapwright-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tapwright
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Let LLM agents test apps on your real iPhone, from Windows, Linux or macOS. No Mac needed.
|
|
5
|
+
Author: bo5amis
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/bo5amis/tapwright
|
|
8
|
+
Project-URL: Documentation, https://github.com/bo5amis/tapwright/tree/main/docs
|
|
9
|
+
Project-URL: Issues, https://github.com/bo5amis/tapwright/issues
|
|
10
|
+
Keywords: iphone,ios,testing,qa,mcp,ai-agents,webdriveragent,xctest,automation
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: pymobiledevice3<12,>=11.50
|
|
22
|
+
Requires-Dist: mcp>=2.3
|
|
23
|
+
Requires-Dist: anyio>=4.5
|
|
24
|
+
Requires-Dist: starlette>=1.0
|
|
25
|
+
Requires-Dist: uvicorn>=0.30
|
|
26
|
+
Requires-Dist: wsproto>=1.2
|
|
27
|
+
Requires-Dist: pillow
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest; extra == "dev"
|
|
30
|
+
Requires-Dist: pyflakes; extra == "dev"
|
|
31
|
+
Provides-Extra: video
|
|
32
|
+
Requires-Dist: av>=14; extra == "video"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# Tapwright
|
|
36
|
+
|
|
37
|
+
**Playwright-style testing on your real iPhone, driven by AI agents. From Windows, Linux or macOS.**
|
|
38
|
+
|
|
39
|
+
Tapwright gives an agent (Claude, Codex, anything that speaks MCP) eyes and hands on your
|
|
40
|
+
iPhone: it sees the screen as a list of elements plus a screenshot, and taps, types,
|
|
41
|
+
scrolls, swipes, answers permission prompts and opens deep links. It works with any app
|
|
42
|
+
on the phone: your TestFlight builds, development builds, the App Store version.
|
|
43
|
+
Nothing is added to your app, and no Mac or Xcode is needed.
|
|
44
|
+
|
|
45
|
+
<p align="center">
|
|
46
|
+
<img src="https://raw.githubusercontent.com/bo5amis/tapwright/main/docs/media/hero.gif" width="320" alt="A Claude agent tests a demo app's sign-up on a real iPhone: it types an invalid email, the app accepts it, then it plans a trip and answers iOS's notification prompt. Each step is captioned under the phone.">
|
|
47
|
+
</p>
|
|
48
|
+
<p align="center"><sub>A Claude agent given one sentence ("test sign-up and a first trip the way a new user would") on a real
|
|
49
|
+
iPhone, through Tapwright. It finds the planted bug (the app accepts "abc" as an email) and answers iOS's own
|
|
50
|
+
permission prompt. Real time, with the agent's thinking pauses shortened;
|
|
51
|
+
<a href="https://github.com/bo5amis/tapwright/blob/main/docs/media/hero-landscape.mp4">with the live view around it</a>.</sub></p>
|
|
52
|
+
|
|
53
|
+
> [!NOTE]
|
|
54
|
+
> Version 1 is for testing and QA on a **test phone that sits next to your computer**, over
|
|
55
|
+
> USB or the same Wi-Fi. It is measured on one phone (an iPhone 14 Pro Max on iOS 26) from
|
|
56
|
+
> one Windows PC; see [Status](#status).
|
|
57
|
+
|
|
58
|
+
## How it works
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
your computer your iPhone
|
|
62
|
+
┌───────────────────────────┐ developer tunnel ┌──────────────────────────┐
|
|
63
|
+
│ agent ── MCP ── tapwright │ (USB, Wi-Fi, or any address │ WebDriverAgent │
|
|
64
|
+
│ │ that reaches the phone) │ (Apple's UI automation) │
|
|
65
|
+
│ pymobiledevice3 ────────┼─────────────────────────────▶│ taps, types, reads the │
|
|
66
|
+
└───────────────────────────┘ │ screen of any app │
|
|
67
|
+
└──────────────────────────┘
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Apple's UI-testing framework (XCTest) can drive any app on a phone in Developer Mode, as
|
|
71
|
+
long as a computer holds a developer connection to it. Tapwright runs
|
|
72
|
+
[WebDriverAgent](https://github.com/appium/WebDriverAgent) (the XCTest runner behind
|
|
73
|
+
Appium) on your phone, signed with your developer identity, and talks to it through
|
|
74
|
+
[pymobiledevice3](https://github.com/doronz88/pymobiledevice3)'s tunnel. That tunnel needs
|
|
75
|
+
no admin rights and works on any OS.
|
|
76
|
+
|
|
77
|
+
Tapwright drives **the whole phone from the outside** (any app, system alerts, the home
|
|
78
|
+
screen). Its sibling project [Tapline](https://github.com/bo5amis/tapline) drives **your own
|
|
79
|
+
React Native app from the inside**, through a small SDK in the app.
|
|
80
|
+
|
|
81
|
+
## You need
|
|
82
|
+
|
|
83
|
+
- An iPhone on iOS 17 or later, and a USB cable for the one-time setup.
|
|
84
|
+
- A computer with Python 3.10+. On Windows, install **Apple Devices** from the Microsoft
|
|
85
|
+
Store (it provides the USB connection); on Linux, `usbmuxd`.
|
|
86
|
+
- An Apple Developer Program membership, for an App Store Connect API key (setup then
|
|
87
|
+
creates the certificate and profile itself). Or bring your own Apple Development
|
|
88
|
+
certificate (.p12) and a development profile for a wildcard App ID that includes the
|
|
89
|
+
phone.
|
|
90
|
+
|
|
91
|
+
## Quick start
|
|
92
|
+
|
|
93
|
+
```sh
|
|
94
|
+
pip install tapwright
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**One-time setup**, phone plugged in and unlocked:
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
tapwright setup --asc-key-id KEY_ID --asc-issuer-id ISSUER_ID --asc-key-file AuthKey_KEY_ID.p8 \
|
|
101
|
+
--bundle-prefix com.yourname.tapwright
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
It pairs with the phone (tap **Trust**), checks Developer Mode, creates a development
|
|
105
|
+
certificate and profile through the API (the private key stays on your computer), signs
|
|
106
|
+
and installs WebDriverAgent, and pairs for network use. Details and the manual option:
|
|
107
|
+
[docs/setup.md](https://github.com/bo5amis/tapwright/blob/main/docs/setup.md).
|
|
108
|
+
|
|
109
|
+
**Try it** on one of your apps:
|
|
110
|
+
|
|
111
|
+
```sh
|
|
112
|
+
tapwright smoke --app com.yourcompany.yourapp
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
It opens the app, prints its first screen as elements, saves a screenshot, and closes it.
|
|
116
|
+
|
|
117
|
+
**Give it to your agent**, naming the apps it may use. Claude Code:
|
|
118
|
+
|
|
119
|
+
```sh
|
|
120
|
+
claude mcp add tapwright -- tapwright mcp --apps com.yourcompany.yourapp
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
- `--apps` takes bundle ids or exact names, comma-separated. The agent is refused every
|
|
124
|
+
other app (`not_allowed`) and is never shown another app's screen, except a permission
|
|
125
|
+
prompt one of your apps raises, which it can see and answer.
|
|
126
|
+
- On a phone kept for testing, with no personal accounts on it, use `--dedicated` instead:
|
|
127
|
+
the agent may then use every app, the Home screen and system prompts.
|
|
128
|
+
- With neither, the agent can do nothing. That is deliberate: the phone may be yours.
|
|
129
|
+
|
|
130
|
+
Other MCP clients: run `tapwright mcp --apps …` (stdio). Then ask, for example: *"Open MyApp
|
|
131
|
+
on my phone, sign up with a test email, and tell me if anything looks broken."*
|
|
132
|
+
[AGENTS.md](https://github.com/bo5amis/tapwright/blob/main/AGENTS.md) is the agent's guide; point your agent at it.
|
|
133
|
+
[docs/quickstart.md](https://github.com/bo5amis/tapwright/blob/main/docs/quickstart.md) walks through a first test.
|
|
134
|
+
|
|
135
|
+
**Give your agent the skill.** It teaches an agent how to test with Tapwright: the loop, how
|
|
136
|
+
to read an answer, what to do on each error, how to report, and the mistakes agents make.
|
|
137
|
+
|
|
138
|
+
```sh
|
|
139
|
+
tapwright skill install # for you: ~/.claude/skills and ~/.agents/skills
|
|
140
|
+
tapwright skill install --project . # for one project
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Claude Code reads `.claude/skills`; Codex and other agents read `.agents/skills`. The skill
|
|
144
|
+
is [skills/tapwright](https://github.com/bo5amis/tapwright/blob/main/skills/tapwright/SKILL.md), in the open
|
|
145
|
+
[Agent Skills](https://agentskills.io) format, so skill installers can take it from this
|
|
146
|
+
repository too.
|
|
147
|
+
|
|
148
|
+
## Tools
|
|
149
|
+
|
|
150
|
+
| Tool | What it does |
|
|
151
|
+
|---|---|
|
|
152
|
+
| `apps`, `open_app`, `close_app` | the apps you installed (TestFlight and development builds included); open one by bundle id or name, or close it |
|
|
153
|
+
| `look` | the screen: every visible element (role, text, id, position in points, what you can do with it) and a screenshot |
|
|
154
|
+
| `tap`, `long_press`, `toggle` | by element `{id}` (`testID`/accessibility identifier), `{label}`, `{text}`, `{ref}` from the last list, or `{x, y}` |
|
|
155
|
+
| `type_text` | into a field (clearing it first by default, or pressing return after) |
|
|
156
|
+
| `scroll`, `swipe`, `back` | lists and pages, to the start, the end, or straight to an element; drags between points; up one screen (an open sheet closes first) |
|
|
157
|
+
| `wait`, `wait_for` | a fixed delay, or until some text, id or label shows up or goes |
|
|
158
|
+
| `alert` | accept, dismiss or read system alerts (permission prompts) |
|
|
159
|
+
| `press_button` | home, volume |
|
|
160
|
+
| `rotate` | turn the phone to portrait or either landscape side, to test an app's landscape layout |
|
|
161
|
+
| `open_url` | deep links |
|
|
162
|
+
| `viewer` | the link to the live view |
|
|
163
|
+
|
|
164
|
+
Every action answers with what changed on the screen, not the whole list, and says how long the
|
|
165
|
+
screen took to settle. Add `screenshot: true` for a picture. `tapwright mcp --screenshots DIR`
|
|
166
|
+
saves the pictures as files instead of sending them. Errors are `Error <code>: <message>`; for
|
|
167
|
+
`not_actionable` the first word says why (`disabled`, `off_screen`, `covered`, `under_tab_bar`,
|
|
168
|
+
`behind_alert`, `no_action`). The tools and their errors are the same as Tapline's:
|
|
169
|
+
[docs/AGENT-TOOLS.md](https://github.com/bo5amis/tapwright/blob/main/docs/AGENT-TOOLS.md).
|
|
170
|
+
|
|
171
|
+
## Live view
|
|
172
|
+
|
|
173
|
+
`tapwright serve` is a small background process that owns the connection to the phone.
|
|
174
|
+
Agent tools, the live view in your browser and any other client go through it, so you and
|
|
175
|
+
an agent can share one phone. Agent tools start it by themselves. To watch:
|
|
176
|
+
|
|
177
|
+
```sh
|
|
178
|
+
tapwright view # opens the live view in your browser
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
<p align="center"><img src="https://raw.githubusercontent.com/bo5amis/tapwright/main/docs/media/still-prompt.png" width="720" alt="The live view: the phone's screen in a drawn phone, iOS asking whether Wayfare may send notifications, and the agent's step captioned under the phone."></p>
|
|
182
|
+
|
|
183
|
+
You see the phone's screen live and can drive it: click to tap, hold to long-press, drag to
|
|
184
|
+
swipe, use the mouse wheel to scroll, and type into the focused field. Each step, the
|
|
185
|
+
agent's or yours, shows in plain words under the phone (`Tap "Save trip"`), and **Steps**
|
|
186
|
+
lists them. A step that couldn't be done shows in red with the reason (`Tap "Save" – covered`). The bar has **Pause agent**; **More** holds Home, volume, the picture quality and
|
|
187
|
+
**Details** (the element names and boxes, for when you're debugging). Your input goes first:
|
|
188
|
+
the agent waits while you drive. The live view is reachable from this computer only, with an
|
|
189
|
+
access token in the link.
|
|
190
|
+
|
|
191
|
+
- Drags play on the phone after you let go: WebDriverAgent can't follow a finger live.
|
|
192
|
+
- The picture is a stream of JPEG frames, about 28 a second while the screen changes and 5
|
|
193
|
+
while it is still.
|
|
194
|
+
- `tapwright serve --video` plays the phone's own video instead, 60 frames a second, with
|
|
195
|
+
nothing installed on the phone. The phone and the computer must be on the same Wi-Fi, and
|
|
196
|
+
the computer needs an AirPlay receiver: see [docs/setup.md](https://github.com/bo5amis/tapwright/blob/main/docs/setup.md).
|
|
197
|
+
- The volume buttons press the phone's real buttons.
|
|
198
|
+
|
|
199
|
+
Other tools can use the same daemon: the agent tools over HTTP (`POST /api/<tool>`), and the
|
|
200
|
+
live view protocol in [docs/VIEWER-PROTOCOL.md](https://github.com/bo5amis/tapwright/blob/main/docs/VIEWER-PROTOCOL.md).
|
|
201
|
+
|
|
202
|
+
## Without the cable
|
|
203
|
+
|
|
204
|
+
After setup, unplug: on the same Wi-Fi, Tapwright finds the phone by itself, or give its
|
|
205
|
+
address with `tapwright serve --phone 192.168.x.y`. Over Wi-Fi every step is slower than
|
|
206
|
+
over USB (see [Status](#status)); for long unattended runs, keep the cable in.
|
|
207
|
+
|
|
208
|
+
Reaching the phone from another network (a VPN such as Tailscale, or cellular) is not part
|
|
209
|
+
of version 1. `tapwright probe --host <address> --watch` exists to try it; treat it as
|
|
210
|
+
experimental.
|
|
211
|
+
|
|
212
|
+
## Limits
|
|
213
|
+
|
|
214
|
+
- Apple's automation sees what accessibility sees. Give important views accessibility
|
|
215
|
+
identifiers (`testID` in React Native) and labels; games and custom-drawn screens show
|
|
216
|
+
little beyond their screenshot.
|
|
217
|
+
- Face ID, Touch ID and the passcode can't be automated; unlock the phone yourself.
|
|
218
|
+
- Before it lets a UI test start, iOS may ask for the passcode ("Enable UI Automation").
|
|
219
|
+
Tapwright waits and tells the agent. Turn on Settings > Developer > Enable UI Automation
|
|
220
|
+
on a test device, or an unattended run stops there.
|
|
221
|
+
- A test phone is the intended setup. If you use a phone with your own accounts on it, an
|
|
222
|
+
agent gets only the apps you name, and that is enforced; inside those apps, the agent guide
|
|
223
|
+
tells agents to stay away from purchases, sending, deleting and account changes unless you
|
|
224
|
+
asked for exactly that: this part is guidance, not enforcement.
|
|
225
|
+
- Notification banners cover the top of the screen for a few seconds. Tapwright waits them
|
|
226
|
+
out and won't touch under one, which can slow a run on a phone that gets notifications.
|
|
227
|
+
Don't silence them with a Focus: iOS shares a Focus with your other devices. On a test phone,
|
|
228
|
+
switch off Allow Notifications per app in Settings > Notifications instead; that stays on the
|
|
229
|
+
one phone.
|
|
230
|
+
- Landscape: `rotate` turns the phone; the live view, screenshots and touches follow it (checked
|
|
231
|
+
on both sides: taps, typing, scrolling, the keyboard, a live-view tap).
|
|
232
|
+
- AssistiveTouch: with a mouse or other pointer connected, iOS shows a cursor, and a tap on
|
|
233
|
+
that cursor opens the AssistiveTouch menu (Home, Notification Center, ...). Neither the cursor
|
|
234
|
+
nor the menu is in what Tapwright reads, so a tap there looks like "No visible change", and the
|
|
235
|
+
next tap only closes the menu. On a test phone, disconnect the pointer or turn AssistiveTouch
|
|
236
|
+
off while an agent drives it.
|
|
237
|
+
|
|
238
|
+
## Status
|
|
239
|
+
|
|
240
|
+
| | |
|
|
241
|
+
|---|---|
|
|
242
|
+
| Works on our phone | setup from Windows; USB and Wi-Fi; every tool; the live view as JPEG frames and as video; the app list (`--apps`) and `--dedicated`; landscape, on both sides |
|
|
243
|
+
| Measured over USB | a tap, with its answer: 0.65 s; a look: 0.14 s; a toggle 0.66 s; typing 0.73 s; a scroll 1.46 s; a person's tap in the live view shows after 0.23 s; the reliability suite with no misses |
|
|
244
|
+
| Measured over Wi-Fi | slower: a tap 1.0 s, a look 0.4 s, typing 1.3 s; the reliability suite with no misses |
|
|
245
|
+
| Agents | a first-time agent with the skill tested the whole lab charter in 44 calls (40 with Tapline connected too) |
|
|
246
|
+
| Not done | over Wi-Fi, a tap (1.03 s) and typing (1.33 s) within Wi-Fi's own bar of 1.0 s (the toggle meets it), and a scroll within 1.5 s; a second phone or an iPad; a run by anyone but the authors |
|
|
247
|
+
|
|
248
|
+
Every number, how it was measured and how to run it yourself: [docs/benchmarks.md](https://github.com/bo5amis/tapwright/blob/main/docs/benchmarks.md).
|
|
249
|
+
The bar they are held against: [docs/QUALITY-BAR.md](https://github.com/bo5amis/tapwright/blob/main/docs/QUALITY-BAR.md).
|
|
250
|
+
|
|
251
|
+
## License
|
|
252
|
+
|
|
253
|
+
MIT. WebDriverAgent (BSD) and rcodesign (MPL-2.0) are downloaded at setup, not included.
|
|
254
|
+
Tapwright is not affiliated with Microsoft's Playwright or with Apple.
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# Tapwright
|
|
2
|
+
|
|
3
|
+
**Playwright-style testing on your real iPhone, driven by AI agents. From Windows, Linux or macOS.**
|
|
4
|
+
|
|
5
|
+
Tapwright gives an agent (Claude, Codex, anything that speaks MCP) eyes and hands on your
|
|
6
|
+
iPhone: it sees the screen as a list of elements plus a screenshot, and taps, types,
|
|
7
|
+
scrolls, swipes, answers permission prompts and opens deep links. It works with any app
|
|
8
|
+
on the phone: your TestFlight builds, development builds, the App Store version.
|
|
9
|
+
Nothing is added to your app, and no Mac or Xcode is needed.
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<img src="https://raw.githubusercontent.com/bo5amis/tapwright/main/docs/media/hero.gif" width="320" alt="A Claude agent tests a demo app's sign-up on a real iPhone: it types an invalid email, the app accepts it, then it plans a trip and answers iOS's notification prompt. Each step is captioned under the phone.">
|
|
13
|
+
</p>
|
|
14
|
+
<p align="center"><sub>A Claude agent given one sentence ("test sign-up and a first trip the way a new user would") on a real
|
|
15
|
+
iPhone, through Tapwright. It finds the planted bug (the app accepts "abc" as an email) and answers iOS's own
|
|
16
|
+
permission prompt. Real time, with the agent's thinking pauses shortened;
|
|
17
|
+
<a href="https://github.com/bo5amis/tapwright/blob/main/docs/media/hero-landscape.mp4">with the live view around it</a>.</sub></p>
|
|
18
|
+
|
|
19
|
+
> [!NOTE]
|
|
20
|
+
> Version 1 is for testing and QA on a **test phone that sits next to your computer**, over
|
|
21
|
+
> USB or the same Wi-Fi. It is measured on one phone (an iPhone 14 Pro Max on iOS 26) from
|
|
22
|
+
> one Windows PC; see [Status](#status).
|
|
23
|
+
|
|
24
|
+
## How it works
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
your computer your iPhone
|
|
28
|
+
┌───────────────────────────┐ developer tunnel ┌──────────────────────────┐
|
|
29
|
+
│ agent ── MCP ── tapwright │ (USB, Wi-Fi, or any address │ WebDriverAgent │
|
|
30
|
+
│ │ that reaches the phone) │ (Apple's UI automation) │
|
|
31
|
+
│ pymobiledevice3 ────────┼─────────────────────────────▶│ taps, types, reads the │
|
|
32
|
+
└───────────────────────────┘ │ screen of any app │
|
|
33
|
+
└──────────────────────────┘
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Apple's UI-testing framework (XCTest) can drive any app on a phone in Developer Mode, as
|
|
37
|
+
long as a computer holds a developer connection to it. Tapwright runs
|
|
38
|
+
[WebDriverAgent](https://github.com/appium/WebDriverAgent) (the XCTest runner behind
|
|
39
|
+
Appium) on your phone, signed with your developer identity, and talks to it through
|
|
40
|
+
[pymobiledevice3](https://github.com/doronz88/pymobiledevice3)'s tunnel. That tunnel needs
|
|
41
|
+
no admin rights and works on any OS.
|
|
42
|
+
|
|
43
|
+
Tapwright drives **the whole phone from the outside** (any app, system alerts, the home
|
|
44
|
+
screen). Its sibling project [Tapline](https://github.com/bo5amis/tapline) drives **your own
|
|
45
|
+
React Native app from the inside**, through a small SDK in the app.
|
|
46
|
+
|
|
47
|
+
## You need
|
|
48
|
+
|
|
49
|
+
- An iPhone on iOS 17 or later, and a USB cable for the one-time setup.
|
|
50
|
+
- A computer with Python 3.10+. On Windows, install **Apple Devices** from the Microsoft
|
|
51
|
+
Store (it provides the USB connection); on Linux, `usbmuxd`.
|
|
52
|
+
- An Apple Developer Program membership, for an App Store Connect API key (setup then
|
|
53
|
+
creates the certificate and profile itself). Or bring your own Apple Development
|
|
54
|
+
certificate (.p12) and a development profile for a wildcard App ID that includes the
|
|
55
|
+
phone.
|
|
56
|
+
|
|
57
|
+
## Quick start
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
pip install tapwright
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
**One-time setup**, phone plugged in and unlocked:
|
|
64
|
+
|
|
65
|
+
```sh
|
|
66
|
+
tapwright setup --asc-key-id KEY_ID --asc-issuer-id ISSUER_ID --asc-key-file AuthKey_KEY_ID.p8 \
|
|
67
|
+
--bundle-prefix com.yourname.tapwright
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
It pairs with the phone (tap **Trust**), checks Developer Mode, creates a development
|
|
71
|
+
certificate and profile through the API (the private key stays on your computer), signs
|
|
72
|
+
and installs WebDriverAgent, and pairs for network use. Details and the manual option:
|
|
73
|
+
[docs/setup.md](https://github.com/bo5amis/tapwright/blob/main/docs/setup.md).
|
|
74
|
+
|
|
75
|
+
**Try it** on one of your apps:
|
|
76
|
+
|
|
77
|
+
```sh
|
|
78
|
+
tapwright smoke --app com.yourcompany.yourapp
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
It opens the app, prints its first screen as elements, saves a screenshot, and closes it.
|
|
82
|
+
|
|
83
|
+
**Give it to your agent**, naming the apps it may use. Claude Code:
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
claude mcp add tapwright -- tapwright mcp --apps com.yourcompany.yourapp
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- `--apps` takes bundle ids or exact names, comma-separated. The agent is refused every
|
|
90
|
+
other app (`not_allowed`) and is never shown another app's screen, except a permission
|
|
91
|
+
prompt one of your apps raises, which it can see and answer.
|
|
92
|
+
- On a phone kept for testing, with no personal accounts on it, use `--dedicated` instead:
|
|
93
|
+
the agent may then use every app, the Home screen and system prompts.
|
|
94
|
+
- With neither, the agent can do nothing. That is deliberate: the phone may be yours.
|
|
95
|
+
|
|
96
|
+
Other MCP clients: run `tapwright mcp --apps …` (stdio). Then ask, for example: *"Open MyApp
|
|
97
|
+
on my phone, sign up with a test email, and tell me if anything looks broken."*
|
|
98
|
+
[AGENTS.md](https://github.com/bo5amis/tapwright/blob/main/AGENTS.md) is the agent's guide; point your agent at it.
|
|
99
|
+
[docs/quickstart.md](https://github.com/bo5amis/tapwright/blob/main/docs/quickstart.md) walks through a first test.
|
|
100
|
+
|
|
101
|
+
**Give your agent the skill.** It teaches an agent how to test with Tapwright: the loop, how
|
|
102
|
+
to read an answer, what to do on each error, how to report, and the mistakes agents make.
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
tapwright skill install # for you: ~/.claude/skills and ~/.agents/skills
|
|
106
|
+
tapwright skill install --project . # for one project
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Claude Code reads `.claude/skills`; Codex and other agents read `.agents/skills`. The skill
|
|
110
|
+
is [skills/tapwright](https://github.com/bo5amis/tapwright/blob/main/skills/tapwright/SKILL.md), in the open
|
|
111
|
+
[Agent Skills](https://agentskills.io) format, so skill installers can take it from this
|
|
112
|
+
repository too.
|
|
113
|
+
|
|
114
|
+
## Tools
|
|
115
|
+
|
|
116
|
+
| Tool | What it does |
|
|
117
|
+
|---|---|
|
|
118
|
+
| `apps`, `open_app`, `close_app` | the apps you installed (TestFlight and development builds included); open one by bundle id or name, or close it |
|
|
119
|
+
| `look` | the screen: every visible element (role, text, id, position in points, what you can do with it) and a screenshot |
|
|
120
|
+
| `tap`, `long_press`, `toggle` | by element `{id}` (`testID`/accessibility identifier), `{label}`, `{text}`, `{ref}` from the last list, or `{x, y}` |
|
|
121
|
+
| `type_text` | into a field (clearing it first by default, or pressing return after) |
|
|
122
|
+
| `scroll`, `swipe`, `back` | lists and pages, to the start, the end, or straight to an element; drags between points; up one screen (an open sheet closes first) |
|
|
123
|
+
| `wait`, `wait_for` | a fixed delay, or until some text, id or label shows up or goes |
|
|
124
|
+
| `alert` | accept, dismiss or read system alerts (permission prompts) |
|
|
125
|
+
| `press_button` | home, volume |
|
|
126
|
+
| `rotate` | turn the phone to portrait or either landscape side, to test an app's landscape layout |
|
|
127
|
+
| `open_url` | deep links |
|
|
128
|
+
| `viewer` | the link to the live view |
|
|
129
|
+
|
|
130
|
+
Every action answers with what changed on the screen, not the whole list, and says how long the
|
|
131
|
+
screen took to settle. Add `screenshot: true` for a picture. `tapwright mcp --screenshots DIR`
|
|
132
|
+
saves the pictures as files instead of sending them. Errors are `Error <code>: <message>`; for
|
|
133
|
+
`not_actionable` the first word says why (`disabled`, `off_screen`, `covered`, `under_tab_bar`,
|
|
134
|
+
`behind_alert`, `no_action`). The tools and their errors are the same as Tapline's:
|
|
135
|
+
[docs/AGENT-TOOLS.md](https://github.com/bo5amis/tapwright/blob/main/docs/AGENT-TOOLS.md).
|
|
136
|
+
|
|
137
|
+
## Live view
|
|
138
|
+
|
|
139
|
+
`tapwright serve` is a small background process that owns the connection to the phone.
|
|
140
|
+
Agent tools, the live view in your browser and any other client go through it, so you and
|
|
141
|
+
an agent can share one phone. Agent tools start it by themselves. To watch:
|
|
142
|
+
|
|
143
|
+
```sh
|
|
144
|
+
tapwright view # opens the live view in your browser
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
<p align="center"><img src="https://raw.githubusercontent.com/bo5amis/tapwright/main/docs/media/still-prompt.png" width="720" alt="The live view: the phone's screen in a drawn phone, iOS asking whether Wayfare may send notifications, and the agent's step captioned under the phone."></p>
|
|
148
|
+
|
|
149
|
+
You see the phone's screen live and can drive it: click to tap, hold to long-press, drag to
|
|
150
|
+
swipe, use the mouse wheel to scroll, and type into the focused field. Each step, the
|
|
151
|
+
agent's or yours, shows in plain words under the phone (`Tap "Save trip"`), and **Steps**
|
|
152
|
+
lists them. A step that couldn't be done shows in red with the reason (`Tap "Save" – covered`). The bar has **Pause agent**; **More** holds Home, volume, the picture quality and
|
|
153
|
+
**Details** (the element names and boxes, for when you're debugging). Your input goes first:
|
|
154
|
+
the agent waits while you drive. The live view is reachable from this computer only, with an
|
|
155
|
+
access token in the link.
|
|
156
|
+
|
|
157
|
+
- Drags play on the phone after you let go: WebDriverAgent can't follow a finger live.
|
|
158
|
+
- The picture is a stream of JPEG frames, about 28 a second while the screen changes and 5
|
|
159
|
+
while it is still.
|
|
160
|
+
- `tapwright serve --video` plays the phone's own video instead, 60 frames a second, with
|
|
161
|
+
nothing installed on the phone. The phone and the computer must be on the same Wi-Fi, and
|
|
162
|
+
the computer needs an AirPlay receiver: see [docs/setup.md](https://github.com/bo5amis/tapwright/blob/main/docs/setup.md).
|
|
163
|
+
- The volume buttons press the phone's real buttons.
|
|
164
|
+
|
|
165
|
+
Other tools can use the same daemon: the agent tools over HTTP (`POST /api/<tool>`), and the
|
|
166
|
+
live view protocol in [docs/VIEWER-PROTOCOL.md](https://github.com/bo5amis/tapwright/blob/main/docs/VIEWER-PROTOCOL.md).
|
|
167
|
+
|
|
168
|
+
## Without the cable
|
|
169
|
+
|
|
170
|
+
After setup, unplug: on the same Wi-Fi, Tapwright finds the phone by itself, or give its
|
|
171
|
+
address with `tapwright serve --phone 192.168.x.y`. Over Wi-Fi every step is slower than
|
|
172
|
+
over USB (see [Status](#status)); for long unattended runs, keep the cable in.
|
|
173
|
+
|
|
174
|
+
Reaching the phone from another network (a VPN such as Tailscale, or cellular) is not part
|
|
175
|
+
of version 1. `tapwright probe --host <address> --watch` exists to try it; treat it as
|
|
176
|
+
experimental.
|
|
177
|
+
|
|
178
|
+
## Limits
|
|
179
|
+
|
|
180
|
+
- Apple's automation sees what accessibility sees. Give important views accessibility
|
|
181
|
+
identifiers (`testID` in React Native) and labels; games and custom-drawn screens show
|
|
182
|
+
little beyond their screenshot.
|
|
183
|
+
- Face ID, Touch ID and the passcode can't be automated; unlock the phone yourself.
|
|
184
|
+
- Before it lets a UI test start, iOS may ask for the passcode ("Enable UI Automation").
|
|
185
|
+
Tapwright waits and tells the agent. Turn on Settings > Developer > Enable UI Automation
|
|
186
|
+
on a test device, or an unattended run stops there.
|
|
187
|
+
- A test phone is the intended setup. If you use a phone with your own accounts on it, an
|
|
188
|
+
agent gets only the apps you name, and that is enforced; inside those apps, the agent guide
|
|
189
|
+
tells agents to stay away from purchases, sending, deleting and account changes unless you
|
|
190
|
+
asked for exactly that: this part is guidance, not enforcement.
|
|
191
|
+
- Notification banners cover the top of the screen for a few seconds. Tapwright waits them
|
|
192
|
+
out and won't touch under one, which can slow a run on a phone that gets notifications.
|
|
193
|
+
Don't silence them with a Focus: iOS shares a Focus with your other devices. On a test phone,
|
|
194
|
+
switch off Allow Notifications per app in Settings > Notifications instead; that stays on the
|
|
195
|
+
one phone.
|
|
196
|
+
- Landscape: `rotate` turns the phone; the live view, screenshots and touches follow it (checked
|
|
197
|
+
on both sides: taps, typing, scrolling, the keyboard, a live-view tap).
|
|
198
|
+
- AssistiveTouch: with a mouse or other pointer connected, iOS shows a cursor, and a tap on
|
|
199
|
+
that cursor opens the AssistiveTouch menu (Home, Notification Center, ...). Neither the cursor
|
|
200
|
+
nor the menu is in what Tapwright reads, so a tap there looks like "No visible change", and the
|
|
201
|
+
next tap only closes the menu. On a test phone, disconnect the pointer or turn AssistiveTouch
|
|
202
|
+
off while an agent drives it.
|
|
203
|
+
|
|
204
|
+
## Status
|
|
205
|
+
|
|
206
|
+
| | |
|
|
207
|
+
|---|---|
|
|
208
|
+
| Works on our phone | setup from Windows; USB and Wi-Fi; every tool; the live view as JPEG frames and as video; the app list (`--apps`) and `--dedicated`; landscape, on both sides |
|
|
209
|
+
| Measured over USB | a tap, with its answer: 0.65 s; a look: 0.14 s; a toggle 0.66 s; typing 0.73 s; a scroll 1.46 s; a person's tap in the live view shows after 0.23 s; the reliability suite with no misses |
|
|
210
|
+
| Measured over Wi-Fi | slower: a tap 1.0 s, a look 0.4 s, typing 1.3 s; the reliability suite with no misses |
|
|
211
|
+
| Agents | a first-time agent with the skill tested the whole lab charter in 44 calls (40 with Tapline connected too) |
|
|
212
|
+
| Not done | over Wi-Fi, a tap (1.03 s) and typing (1.33 s) within Wi-Fi's own bar of 1.0 s (the toggle meets it), and a scroll within 1.5 s; a second phone or an iPad; a run by anyone but the authors |
|
|
213
|
+
|
|
214
|
+
Every number, how it was measured and how to run it yourself: [docs/benchmarks.md](https://github.com/bo5amis/tapwright/blob/main/docs/benchmarks.md).
|
|
215
|
+
The bar they are held against: [docs/QUALITY-BAR.md](https://github.com/bo5amis/tapwright/blob/main/docs/QUALITY-BAR.md).
|
|
216
|
+
|
|
217
|
+
## License
|
|
218
|
+
|
|
219
|
+
MIT. WebDriverAgent (BSD) and rcodesign (MPL-2.0) are downloaded at setup, not included.
|
|
220
|
+
Tapwright is not affiliated with Microsoft's Playwright or with Apple.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tapwright"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Let LLM agents test apps on your real iPhone, from Windows, Linux or macOS. No Mac needed."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "bo5amis" }]
|
|
13
|
+
keywords = ["iphone", "ios", "testing", "qa", "mcp", "ai-agents", "webdriveragent", "xctest", "automation"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Operating System :: Microsoft :: Windows",
|
|
18
|
+
"Operating System :: POSIX :: Linux",
|
|
19
|
+
"Operating System :: MacOS",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: Software Development :: Testing",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"pymobiledevice3>=11.50,<12", # Apple's device protocols; device.py relies on its user-space tunnel
|
|
25
|
+
"mcp>=2.3",
|
|
26
|
+
"anyio>=4.5", # runs everything on the event loop pymobiledevice3 needs (device.run)
|
|
27
|
+
"starlette>=1.0", # the daemon's HTTP and WebSocket app
|
|
28
|
+
"uvicorn>=0.30",
|
|
29
|
+
"wsproto>=1.2", # uvicorn's WebSocket implementation (server.py uses ws="wsproto")
|
|
30
|
+
"pillow",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/bo5amis/tapwright"
|
|
35
|
+
Documentation = "https://github.com/bo5amis/tapwright/tree/main/docs"
|
|
36
|
+
Issues = "https://github.com/bo5amis/tapwright/issues"
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
dev = ["pytest", "pyflakes"]
|
|
40
|
+
video = ["av>=14"] # the H.264 live view (/video): decode the AirPlay stream, encode it again
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
tapwright = "tapwright.cli:main"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools]
|
|
46
|
+
packages = ["tapwright"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.package-data]
|
|
49
|
+
tapwright = ["viewer.html", "skill/*.md", "skill/references/*.md", "skill/agents/*.yaml"]
|