phone-harness 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phone_harness-0.1.1/.gitignore +5 -0
- phone_harness-0.1.1/LICENSE +21 -0
- phone_harness-0.1.1/PKG-INFO +152 -0
- phone_harness-0.1.1/README.md +130 -0
- phone_harness-0.1.1/SKILL.md +102 -0
- phone_harness-0.1.1/install.md +93 -0
- phone_harness-0.1.1/pyproject.toml +45 -0
- phone_harness-0.1.1/src/phone_harness/__init__.py +0 -0
- phone_harness-0.1.1/src/phone_harness/admin.py +70 -0
- phone_harness-0.1.1/src/phone_harness/background.py +261 -0
- phone_harness-0.1.1/src/phone_harness/helpers.py +370 -0
- phone_harness-0.1.1/src/phone_harness/mirror.py +259 -0
- phone_harness-0.1.1/src/phone_harness/ocr.py +56 -0
- phone_harness-0.1.1/src/phone_harness/run.py +53 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 shawn pana
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: phone-harness
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Control a real iPhone through macOS iPhone Mirroring: OCR eyes, CGEvent hands
|
|
5
|
+
Project-URL: Homepage, https://phone-harness.com
|
|
6
|
+
Project-URL: Repository, https://github.com/ShawnPana/phone-harness
|
|
7
|
+
Project-URL: Issues, https://github.com/ShawnPana/phone-harness/issues
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agent,automation,iphone,iphone-mirroring,macos
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: MacOS X
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Software Development :: Testing
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: pyobjc-framework-applicationservices
|
|
18
|
+
Requires-Dist: pyobjc-framework-cocoa
|
|
19
|
+
Requires-Dist: pyobjc-framework-quartz
|
|
20
|
+
Requires-Dist: pyobjc-framework-vision
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# Phone Harness ๐ฑ
|
|
24
|
+
|
|
25
|
+
**[phone-harness](https://phone-harness.com)** ยท let your agent control your phone.
|
|
26
|
+
|
|
27
|
+
Connect an AI agent โ Claude Code, Codex, or any LLM โ directly to your real
|
|
28
|
+
iPhone with a thin, editable harness. No jailbreak, no Xcode, no WebDriverAgent.
|
|
29
|
+
|
|
30
|
+
The Mac's iPhone Mirroring window is the whole transport: `screencapture` +
|
|
31
|
+
Vision-framework OCR for eyes, HID-level CGEvents for hands. Nothing between the
|
|
32
|
+
agent and the phone. The agent writes what's missing during execution in
|
|
33
|
+
`agent-workspace/agent_helpers.py`.
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
โ agent: wants to open Weather
|
|
37
|
+
โ
|
|
38
|
+
โ ocr() โ "Weather" at (400, 468)
|
|
39
|
+
โ
|
|
40
|
+
โ tap(400, 468) โ wait_stable() โ ocr() confirms the forecast
|
|
41
|
+
โ done
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
**Your phone, driven by an agent.**
|
|
45
|
+
|
|
46
|
+
## Setup prompt
|
|
47
|
+
|
|
48
|
+
Paste into Claude Code or Codex:
|
|
49
|
+
|
|
50
|
+
```text
|
|
51
|
+
Set up phone-harness for me. Clone https://github.com/ShawnPana/phone-harness
|
|
52
|
+
into ~/.phone-harness (its canonical home) and read `install.md` first to install
|
|
53
|
+
it and connect it to my real iPhone
|
|
54
|
+
through the macOS iPhone Mirroring app โ install it so `phone-harness` is a
|
|
55
|
+
command on my PATH, and register it as an agent skill named phone-harness using
|
|
56
|
+
`phone-harness skill` as the body, so you reach for it automatically. Then read
|
|
57
|
+
`SKILL.md` for normal usage, and always read `src/phone_harness/helpers.py`
|
|
58
|
+
because that is where the functions are. Whenever you capture or verify the
|
|
59
|
+
screen, activate the iPhone Mirroring window so I can see what you're doing on
|
|
60
|
+
the phone.
|
|
61
|
+
|
|
62
|
+
Setup needs two things only I can do: pairing iPhone Mirroring with my phone
|
|
63
|
+
once, and granting the terminal Accessibility + Screen Recording in System
|
|
64
|
+
Settings โ walk me through those and wait for me. Verify with
|
|
65
|
+
`./phone-harness --doctor`.
|
|
66
|
+
|
|
67
|
+
After it's installed, as a quick demo that interaction works, go to my Home
|
|
68
|
+
Screen and โ if the phone is connected and unlocked โ ask me whether you should
|
|
69
|
+
open the Weather app as a harmless test; only open it if I say yes. If the
|
|
70
|
+
session is paused or the phone is locked, just tell me the doctor status instead.
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The agent will walk you through the two things only you can do: **pairing**
|
|
74
|
+
iPhone Mirroring with your phone once (the pairing prompts need the physical
|
|
75
|
+
phone), and granting the terminal **Accessibility** (taps & keystrokes) and
|
|
76
|
+
**Screen Recording** (seeing the phone) in System Settings โ Privacy &
|
|
77
|
+
Security. Screen Recording takes effect after the terminal restarts;
|
|
78
|
+
Accessibility is immediate. Then `./phone-harness --doctor` verifies the whole
|
|
79
|
+
chain.
|
|
80
|
+
|
|
81
|
+
These are the permissions currently known to be required. A fresh machine may
|
|
82
|
+
prompt for more the first time an action runs โ if `--doctor` passes but taps or
|
|
83
|
+
capture silently do nothing, watch for a macOS permission prompt. See
|
|
84
|
+
[install.md](install.md) for details.
|
|
85
|
+
|
|
86
|
+
## Why this works
|
|
87
|
+
|
|
88
|
+
iPhone Mirroring (macOS Sequoia+) renders the phone as a Mac window and forwards
|
|
89
|
+
real mouse and keyboard input as touches. That gives an agent everything it
|
|
90
|
+
needs for real-device iOS automation:
|
|
91
|
+
|
|
92
|
+
- **See** โ capture just the mirroring window, OCR it with Apple's Vision
|
|
93
|
+
framework: every visible string with a tap-ready coordinate. The poor man's
|
|
94
|
+
DOM.
|
|
95
|
+
- **Act** โ CGEvents posted at the HID tap: taps, long-presses, drags/flicks,
|
|
96
|
+
scroll gestures, unicode typing, and the app's own shortcuts (Cmd+1 Home,
|
|
97
|
+
Cmd+2 App Switcher, Cmd+3 Spotlight).
|
|
98
|
+
- **Verify** โ screenshot again. No DOM means the capture is the ground truth.
|
|
99
|
+
|
|
100
|
+
Things that do NOT work, learned the hard way: AppleScript `click at` (silently
|
|
101
|
+
ignored โ the window is a video stream with no accessibility tree), unicode key
|
|
102
|
+
payloads (mirroring forwards raw HID keycodes, so typing must use keycodes), a
|
|
103
|
+
slow touch-drag (barely moves an iOS list โ use wheel scroll for lists, a fast
|
|
104
|
+
flick for pages), and input while the window isn't frontmost (swallowed).
|
|
105
|
+
|
|
106
|
+
## Usage
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
./phone-harness <<'PY'
|
|
110
|
+
open_app("Notes")
|
|
111
|
+
tap_text("New Note")
|
|
112
|
+
type_text("hello from the harness")
|
|
113
|
+
print([o["text"] for o in ocr()][:10])
|
|
114
|
+
PY
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Day-to-day workflow lives in [SKILL.md](SKILL.md), which [install.md](install.md)
|
|
118
|
+
registers as an agent skill (`phone-harness skill` prints the body) so the agent
|
|
119
|
+
reaches for it on its own.
|
|
120
|
+
|
|
121
|
+
## Architecture
|
|
122
|
+
|
|
123
|
+
- `SKILL.md` โ day-to-day usage (the agent-facing product surface)
|
|
124
|
+
- `install.md` โ permissions bootstrap and troubleshooting
|
|
125
|
+
- `src/phone_harness/` โ protected core (~500 lines):
|
|
126
|
+
- `mirror.py` โ window discovery, focus, capture, CGEvent input
|
|
127
|
+
- `ocr.py` โ Vision-framework text recognition โ screen-point boxes
|
|
128
|
+
- `helpers.py` โ the primitives pre-imported into scripts
|
|
129
|
+
- `admin.py` โ `--doctor`
|
|
130
|
+
- `run.py` โ the CLI (`exec` stdin with helpers in scope)
|
|
131
|
+
- `agent-workspace/agent_helpers.py` โ helper code the agent edits; auto-loaded
|
|
132
|
+
into every script's namespace
|
|
133
|
+
|
|
134
|
+
The mirror transport is stateless (window bounds and captures are re-queried per
|
|
135
|
+
call), so there is no daemon โ every invocation is self-contained.
|
|
136
|
+
|
|
137
|
+
## Development
|
|
138
|
+
|
|
139
|
+
From a checkout, use `./phone-harness` to run the working tree directly:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
./phone-harness <<'PY'
|
|
143
|
+
print(screen_info())
|
|
144
|
+
PY
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Limits
|
|
148
|
+
|
|
149
|
+
- One phone, one session; unlocking the physical phone pauses mirroring.
|
|
150
|
+
- No multi-touch (no pinch), no camera/Face ID flows, DRM video renders black.
|
|
151
|
+
- OCR sees text, not semantics โ unlabeled icons need a screenshot + a
|
|
152
|
+
vision-capable model.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# Phone Harness ๐ฑ
|
|
2
|
+
|
|
3
|
+
**[phone-harness](https://phone-harness.com)** ยท let your agent control your phone.
|
|
4
|
+
|
|
5
|
+
Connect an AI agent โ Claude Code, Codex, or any LLM โ directly to your real
|
|
6
|
+
iPhone with a thin, editable harness. No jailbreak, no Xcode, no WebDriverAgent.
|
|
7
|
+
|
|
8
|
+
The Mac's iPhone Mirroring window is the whole transport: `screencapture` +
|
|
9
|
+
Vision-framework OCR for eyes, HID-level CGEvents for hands. Nothing between the
|
|
10
|
+
agent and the phone. The agent writes what's missing during execution in
|
|
11
|
+
`agent-workspace/agent_helpers.py`.
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
โ agent: wants to open Weather
|
|
15
|
+
โ
|
|
16
|
+
โ ocr() โ "Weather" at (400, 468)
|
|
17
|
+
โ
|
|
18
|
+
โ tap(400, 468) โ wait_stable() โ ocr() confirms the forecast
|
|
19
|
+
โ done
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
**Your phone, driven by an agent.**
|
|
23
|
+
|
|
24
|
+
## Setup prompt
|
|
25
|
+
|
|
26
|
+
Paste into Claude Code or Codex:
|
|
27
|
+
|
|
28
|
+
```text
|
|
29
|
+
Set up phone-harness for me. Clone https://github.com/ShawnPana/phone-harness
|
|
30
|
+
into ~/.phone-harness (its canonical home) and read `install.md` first to install
|
|
31
|
+
it and connect it to my real iPhone
|
|
32
|
+
through the macOS iPhone Mirroring app โ install it so `phone-harness` is a
|
|
33
|
+
command on my PATH, and register it as an agent skill named phone-harness using
|
|
34
|
+
`phone-harness skill` as the body, so you reach for it automatically. Then read
|
|
35
|
+
`SKILL.md` for normal usage, and always read `src/phone_harness/helpers.py`
|
|
36
|
+
because that is where the functions are. Whenever you capture or verify the
|
|
37
|
+
screen, activate the iPhone Mirroring window so I can see what you're doing on
|
|
38
|
+
the phone.
|
|
39
|
+
|
|
40
|
+
Setup needs two things only I can do: pairing iPhone Mirroring with my phone
|
|
41
|
+
once, and granting the terminal Accessibility + Screen Recording in System
|
|
42
|
+
Settings โ walk me through those and wait for me. Verify with
|
|
43
|
+
`./phone-harness --doctor`.
|
|
44
|
+
|
|
45
|
+
After it's installed, as a quick demo that interaction works, go to my Home
|
|
46
|
+
Screen and โ if the phone is connected and unlocked โ ask me whether you should
|
|
47
|
+
open the Weather app as a harmless test; only open it if I say yes. If the
|
|
48
|
+
session is paused or the phone is locked, just tell me the doctor status instead.
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The agent will walk you through the two things only you can do: **pairing**
|
|
52
|
+
iPhone Mirroring with your phone once (the pairing prompts need the physical
|
|
53
|
+
phone), and granting the terminal **Accessibility** (taps & keystrokes) and
|
|
54
|
+
**Screen Recording** (seeing the phone) in System Settings โ Privacy &
|
|
55
|
+
Security. Screen Recording takes effect after the terminal restarts;
|
|
56
|
+
Accessibility is immediate. Then `./phone-harness --doctor` verifies the whole
|
|
57
|
+
chain.
|
|
58
|
+
|
|
59
|
+
These are the permissions currently known to be required. A fresh machine may
|
|
60
|
+
prompt for more the first time an action runs โ if `--doctor` passes but taps or
|
|
61
|
+
capture silently do nothing, watch for a macOS permission prompt. See
|
|
62
|
+
[install.md](install.md) for details.
|
|
63
|
+
|
|
64
|
+
## Why this works
|
|
65
|
+
|
|
66
|
+
iPhone Mirroring (macOS Sequoia+) renders the phone as a Mac window and forwards
|
|
67
|
+
real mouse and keyboard input as touches. That gives an agent everything it
|
|
68
|
+
needs for real-device iOS automation:
|
|
69
|
+
|
|
70
|
+
- **See** โ capture just the mirroring window, OCR it with Apple's Vision
|
|
71
|
+
framework: every visible string with a tap-ready coordinate. The poor man's
|
|
72
|
+
DOM.
|
|
73
|
+
- **Act** โ CGEvents posted at the HID tap: taps, long-presses, drags/flicks,
|
|
74
|
+
scroll gestures, unicode typing, and the app's own shortcuts (Cmd+1 Home,
|
|
75
|
+
Cmd+2 App Switcher, Cmd+3 Spotlight).
|
|
76
|
+
- **Verify** โ screenshot again. No DOM means the capture is the ground truth.
|
|
77
|
+
|
|
78
|
+
Things that do NOT work, learned the hard way: AppleScript `click at` (silently
|
|
79
|
+
ignored โ the window is a video stream with no accessibility tree), unicode key
|
|
80
|
+
payloads (mirroring forwards raw HID keycodes, so typing must use keycodes), a
|
|
81
|
+
slow touch-drag (barely moves an iOS list โ use wheel scroll for lists, a fast
|
|
82
|
+
flick for pages), and input while the window isn't frontmost (swallowed).
|
|
83
|
+
|
|
84
|
+
## Usage
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
./phone-harness <<'PY'
|
|
88
|
+
open_app("Notes")
|
|
89
|
+
tap_text("New Note")
|
|
90
|
+
type_text("hello from the harness")
|
|
91
|
+
print([o["text"] for o in ocr()][:10])
|
|
92
|
+
PY
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Day-to-day workflow lives in [SKILL.md](SKILL.md), which [install.md](install.md)
|
|
96
|
+
registers as an agent skill (`phone-harness skill` prints the body) so the agent
|
|
97
|
+
reaches for it on its own.
|
|
98
|
+
|
|
99
|
+
## Architecture
|
|
100
|
+
|
|
101
|
+
- `SKILL.md` โ day-to-day usage (the agent-facing product surface)
|
|
102
|
+
- `install.md` โ permissions bootstrap and troubleshooting
|
|
103
|
+
- `src/phone_harness/` โ protected core (~500 lines):
|
|
104
|
+
- `mirror.py` โ window discovery, focus, capture, CGEvent input
|
|
105
|
+
- `ocr.py` โ Vision-framework text recognition โ screen-point boxes
|
|
106
|
+
- `helpers.py` โ the primitives pre-imported into scripts
|
|
107
|
+
- `admin.py` โ `--doctor`
|
|
108
|
+
- `run.py` โ the CLI (`exec` stdin with helpers in scope)
|
|
109
|
+
- `agent-workspace/agent_helpers.py` โ helper code the agent edits; auto-loaded
|
|
110
|
+
into every script's namespace
|
|
111
|
+
|
|
112
|
+
The mirror transport is stateless (window bounds and captures are re-queried per
|
|
113
|
+
call), so there is no daemon โ every invocation is self-contained.
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
From a checkout, use `./phone-harness` to run the working tree directly:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
./phone-harness <<'PY'
|
|
121
|
+
print(screen_info())
|
|
122
|
+
PY
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Limits
|
|
126
|
+
|
|
127
|
+
- One phone, one session; unlocking the physical phone pauses mirroring.
|
|
128
|
+
- No multi-touch (no pinch), no camera/Face ID flows, DRM video renders black.
|
|
129
|
+
- OCR sees text, not semantics โ unlabeled icons need a screenshot + a
|
|
130
|
+
vision-capable model.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: phone-harness
|
|
3
|
+
description: "Control the user's iPhone through the Mac's iPhone Mirroring window: open apps, tap, type, swipe, read the screen."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# phone-harness
|
|
7
|
+
|
|
8
|
+
Direct iPhone control via the iPhone Mirroring app โ screenshots + Vision OCR
|
|
9
|
+
for eyes, HID-level CGEvents for hands. For task-specific edits, use
|
|
10
|
+
`agent-workspace/agent_helpers.py`. For setup or permission problems, read
|
|
11
|
+
`install.md`.
|
|
12
|
+
|
|
13
|
+
## When Not to Use
|
|
14
|
+
|
|
15
|
+
If the task is doable on the Mac or the web โ a website, an API, an app with a
|
|
16
|
+
web equivalent โ do it there and leave the phone alone. Use phone-harness only
|
|
17
|
+
when the task genuinely needs the phone: iOS-only apps, things tied to the
|
|
18
|
+
user's phone number or 2FA, testing how something looks on the phone.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
phone-harness <<'PY'
|
|
24
|
+
print(screen_info())
|
|
25
|
+
PY
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
- Invoke as `phone-harness`. Use heredocs for multi-line commands.
|
|
29
|
+
- Helpers are pre-imported. All coordinates are global screen points.
|
|
30
|
+
- `ensure_mirroring()` launches and focuses the window; input helpers focus it
|
|
31
|
+
automatically before posting events.
|
|
32
|
+
|
|
33
|
+
## Screen Workflow
|
|
34
|
+
|
|
35
|
+
- Prefer `ocr()` over eyeballing screenshots: every visible string comes back
|
|
36
|
+
with a tap-ready center point โ `[{text, confidence, x, y, w, h}]`. Filter
|
|
37
|
+
in Python before printing.
|
|
38
|
+
- Tap by label: `tap_text("Weather")`. On failure it raises with what IS
|
|
39
|
+
visible, so read the exception before retrying.
|
|
40
|
+
- Icons without labels: `screenshot()`, view the image, compute the point
|
|
41
|
+
(image px รท scale + window origin โ `screen_info()` has both sizes), then
|
|
42
|
+
`tap(x, y)`.
|
|
43
|
+
- **Verify after every action**: `wait_stable()` then `ocr()`/`screenshot()`.
|
|
44
|
+
There is no DOM to assert against; the capture is the ground truth.
|
|
45
|
+
- Navigation: `home()`, `app_switcher()`, `open_app("Notes")` (Spotlight),
|
|
46
|
+
`swipe("up")`, `scroll()`, `type_text("...")`, `press("return")`,
|
|
47
|
+
`long_press(x, y)`.
|
|
48
|
+
- **Scrolling a list**: use `scroll_collect(extract, key=...)` to walk a list
|
|
49
|
+
to its true end, de-duping as it goes โ it returns `{items, stop, scrolls}`
|
|
50
|
+
where `stop` is `'reached-end'` or `'max-scrolls'`. Use `scroll_until(done)`
|
|
51
|
+
to stop when a predicate on the visible OCR is met. Both decide "done" from
|
|
52
|
+
whether the **screen actually moved**, not from whether your parser found
|
|
53
|
+
new rows โ a dense screen or a missed OCR line will not end the scroll
|
|
54
|
+
early. Each step settles first so lazy-loaded content arrives before the
|
|
55
|
+
movement check. `scroll_screen()` is the single-step primitive if you need
|
|
56
|
+
it. These use wheel scrolling (a slow touch-drag barely moves an iOS list
|
|
57
|
+
and bounces back).
|
|
58
|
+
- Raw Quartz is the escape hatch: `import Quartz` in your script for anything
|
|
59
|
+
the helpers don't cover.
|
|
60
|
+
|
|
61
|
+
## Consent
|
|
62
|
+
|
|
63
|
+
This is the user's real phone. Stop and ask before anything outward-facing or
|
|
64
|
+
hard to reverse: sending a message, posting, purchasing, deleting, changing
|
|
65
|
+
settings. Navigating and reading for the user's own task is fine, but don't
|
|
66
|
+
linger in personal content (Messages, Photos, Mail) beyond what the task needs.
|
|
67
|
+
|
|
68
|
+
## Connection is the user's job
|
|
69
|
+
|
|
70
|
+
The harness never connects the phone for you. Connecting or resuming mirroring
|
|
71
|
+
is a physical action โ opening the app, approving the prompt, and (crucially)
|
|
72
|
+
**locking the iPhone when it says "iPhone in Use"** โ that only the user can do.
|
|
73
|
+
|
|
74
|
+
`ensure_mirroring()` gates every task on this: if the phone isn't connected it
|
|
75
|
+
raises a clear message (call `connection_state()` yourself to check โ
|
|
76
|
+
`ready` / `blocked` / `no-window` / `not-running`). When you hit that:
|
|
77
|
+
|
|
78
|
+
- **STOP and relay the message. Ask the user to connect the phone themselves.**
|
|
79
|
+
- **Never** tap `Connect` / `Continue`, and **never** loop-poll waiting for the
|
|
80
|
+
connection. Tapping Connect while the phone is unlocked does nothing, and
|
|
81
|
+
polling just burns time โ the only fix is the user locking/connecting the
|
|
82
|
+
phone. Retry once *after they confirm they've done it*, not before.
|
|
83
|
+
|
|
84
|
+
## Gotchas
|
|
85
|
+
|
|
86
|
+
- **Unfocused input is swallowed silently.** The window must be frontmost;
|
|
87
|
+
helpers call `activate()` but if a click steals focus mid-task, re-activate.
|
|
88
|
+
- **The window is a video stream.** macOS accessibility sees nothing inside
|
|
89
|
+
it; AppleScript `click at` fails silently. Only HID-level CGEvents work.
|
|
90
|
+
- **The window moves.** Never cache coordinates across calls; `ocr()` and
|
|
91
|
+
`swipe()` re-query bounds every time.
|
|
92
|
+
- **Unlocking the physical phone pauses the session** ("iPhone in Use"). Do not
|
|
93
|
+
tap through the resume screen โ stop and ask the user to lock/connect the
|
|
94
|
+
phone (see "Connection is the user's job").
|
|
95
|
+
- **`type_text` needs an iOS text field focused first** โ tap the field, wait
|
|
96
|
+
for the keyboard, then type.
|
|
97
|
+
- **Home-Screen labels are not tap targets.** `tap_text("Weather")` hits the
|
|
98
|
+
label and nothing happens; the icon is ~35 points above it. Use
|
|
99
|
+
`tap_icon("Weather")` (agent helper) on the Home Screen; `tap_text` works
|
|
100
|
+
fine for in-app buttons and list rows.
|
|
101
|
+
- Mouse taps map to touches 1:1, but there is no multi-touch: no pinch, no
|
|
102
|
+
two-finger gestures.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# phone-harness install
|
|
2
|
+
|
|
3
|
+
Use once. For phone work, read `SKILL.md`.
|
|
4
|
+
|
|
5
|
+
## Requirements
|
|
6
|
+
|
|
7
|
+
- macOS Sequoia+ with iPhone Mirroring paired to the phone (open the app once
|
|
8
|
+
manually to pair โ pairing prompts need the physical phone).
|
|
9
|
+
- Python 3.12+ with pyobjc (`pip install pyobjc-framework-Quartz
|
|
10
|
+
pyobjc-framework-Vision pyobjc-framework-AppKit`).
|
|
11
|
+
- The terminal app needs two permissions in System Settings > Privacy &
|
|
12
|
+
Security. **The toggles require the user:**
|
|
13
|
+
- **Accessibility** โ taps and keystrokes. Takes effect immediately.
|
|
14
|
+
- **Screen Recording** โ seeing the phone. Takes effect after the terminal
|
|
15
|
+
app restarts.
|
|
16
|
+
|
|
17
|
+
Open the panes directly:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
open "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
|
|
21
|
+
open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
> **Heads up โ you may need to grant more than these two.** Accessibility and
|
|
25
|
+
> Screen Recording are the permissions we *know* are required, and they're all
|
|
26
|
+
> `--doctor` currently checks. But this was built on a machine that was already
|
|
27
|
+
> permissive, so a fresh Mac may prompt for additional approvals the first time
|
|
28
|
+
> an action runs. If `--doctor` passes but taps, typing, or capture silently do
|
|
29
|
+
> nothing, watch for a macOS permission prompt and check System Settings >
|
|
30
|
+
> Privacy & Security for a pane asking to approve your terminal. As we pin down
|
|
31
|
+
> exactly which extra permissions a clean install needs, they'll get added to
|
|
32
|
+
> `--doctor` as proper prerequisites.
|
|
33
|
+
|
|
34
|
+
## Fast Path
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
git clone https://github.com/ShawnPana/phone-harness ~/.phone-harness # canonical home
|
|
38
|
+
cd ~/.phone-harness
|
|
39
|
+
pip install pyobjc-framework-Quartz pyobjc-framework-Vision pyobjc-framework-AppKit
|
|
40
|
+
pip install -e . --no-deps # installs the global `phone-harness` command
|
|
41
|
+
|
|
42
|
+
# register as an agent skill so Claude Code / Codex auto-use it (see below)
|
|
43
|
+
mkdir -p ~/.claude/skills/phone-harness
|
|
44
|
+
phone-harness skill > ~/.claude/skills/phone-harness/SKILL.md
|
|
45
|
+
mkdir -p "${CODEX_HOME:-$HOME/.codex}/skills/phone-harness"
|
|
46
|
+
phone-harness skill > "${CODEX_HOME:-$HOME/.codex}/skills/phone-harness/SKILL.md"
|
|
47
|
+
|
|
48
|
+
phone-harness --doctor
|
|
49
|
+
phone-harness <<'PY'
|
|
50
|
+
print(screen_info())
|
|
51
|
+
PY
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
If `screen_info()` prints window bounds, you're done.
|
|
55
|
+
|
|
56
|
+
`~/.phone-harness` is the canonical home โ a hidden folder in your home
|
|
57
|
+
directory (like `~/.oh-my-zsh` or `~/.nvm`), so the code, `helpers.py`,
|
|
58
|
+
`SKILL.md`, and your `agent-workspace/` always live at a path the agent knows,
|
|
59
|
+
on any machine. `pip install -e .` keeps that source editable while giving you a
|
|
60
|
+
`phone-harness` command on PATH, which is what lets the skill call
|
|
61
|
+
`phone-harness` from any directory. Because the install is editable it's bound
|
|
62
|
+
to that location, so keep the folder at `~/.phone-harness` (or re-run `pip
|
|
63
|
+
install -e .` if you relocate it). If your Python can't reach PyPI to resolve the
|
|
64
|
+
pyobjc deps, `--no-deps` skips them (they're installed by the line above).
|
|
65
|
+
|
|
66
|
+
## Register as a skill
|
|
67
|
+
|
|
68
|
+
So the agent reaches for phone-harness on its own, register `SKILL.md` as a
|
|
69
|
+
skill named `phone-harness` with `phone-harness skill` as its body (the Fast
|
|
70
|
+
Path does this for both Claude Code and Codex). The skill's trigger is:
|
|
71
|
+
|
|
72
|
+
```text
|
|
73
|
+
Control the user's iPhone through the Mac's iPhone Mirroring window: open apps,
|
|
74
|
+
tap, type, swipe, read the screen.
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Re-run the `phone-harness skill > โฆ/SKILL.md` lines after pulling updates so the
|
|
78
|
+
registered copy stays current.
|
|
79
|
+
|
|
80
|
+
## If It Fails
|
|
81
|
+
|
|
82
|
+
`--doctor` walks the ladder in order: pyobjc โ Accessibility โ Screen
|
|
83
|
+
Recording โ app installed โ app running โ window found โ capture works โ OCR
|
|
84
|
+
works. Fix the first FAIL; later checks depend on earlier ones.
|
|
85
|
+
|
|
86
|
+
Common cases:
|
|
87
|
+
|
|
88
|
+
- **Capture is blank/black**: Screen Recording was granted but the terminal
|
|
89
|
+
hasn't restarted since.
|
|
90
|
+
- **Window not found**: the phone isn't paired, isn't in range, or iPhone
|
|
91
|
+
Mirroring shows a connect screen โ open the app manually once.
|
|
92
|
+
- **Taps do nothing**: Accessibility missing, or another window stole focus โ
|
|
93
|
+
events land only when the mirroring window is frontmost.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "phone-harness"
|
|
3
|
+
version = "0.1.1"
|
|
4
|
+
description = "Control a real iPhone through macOS iPhone Mirroring: OCR eyes, CGEvent hands"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
keywords = ["iphone", "automation", "agent", "iphone-mirroring", "macos"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 3 - Alpha",
|
|
12
|
+
"Environment :: MacOS X",
|
|
13
|
+
"Operating System :: MacOS",
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Topic :: Software Development :: Testing",
|
|
16
|
+
]
|
|
17
|
+
dependencies = [
|
|
18
|
+
"pyobjc-framework-Quartz",
|
|
19
|
+
"pyobjc-framework-Vision",
|
|
20
|
+
"pyobjc-framework-Cocoa",
|
|
21
|
+
"pyobjc-framework-ApplicationServices",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://phone-harness.com"
|
|
26
|
+
Repository = "https://github.com/ShawnPana/phone-harness"
|
|
27
|
+
Issues = "https://github.com/ShawnPana/phone-harness/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
phone-harness = "phone_harness.run:main"
|
|
31
|
+
|
|
32
|
+
[build-system]
|
|
33
|
+
requires = ["hatchling"]
|
|
34
|
+
build-backend = "hatchling.build"
|
|
35
|
+
|
|
36
|
+
[tool.hatch.build.targets.wheel]
|
|
37
|
+
packages = ["src/phone_harness"]
|
|
38
|
+
|
|
39
|
+
# The skill text ships inside the package so `phone-harness skill` works
|
|
40
|
+
# from an installed wheel, not only from a checkout.
|
|
41
|
+
[tool.hatch.build.targets.wheel.force-include]
|
|
42
|
+
"SKILL.md" = "phone_harness/SKILL.md"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.sdist]
|
|
45
|
+
include = ["src/phone_harness", "SKILL.md", "README.md", "LICENSE", "install.md"]
|
|
File without changes
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Diagnostics: `phone-harness --doctor` walks the permission/session ladder."""
|
|
2
|
+
import os, subprocess, sys, tempfile
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _check(label, ok, hint=""):
|
|
7
|
+
mark = "PASS" if ok else "FAIL"
|
|
8
|
+
print(f" [{mark}] {label}" + (f" โ {hint}" if not ok and hint else ""))
|
|
9
|
+
return ok
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def run_doctor():
|
|
13
|
+
print("phone-harness doctor\n")
|
|
14
|
+
ok = True
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
import Quartz, Vision, AppKit # noqa: F401
|
|
18
|
+
ok &= _check("pyobjc frameworks (Quartz, Vision, AppKit)", True)
|
|
19
|
+
except ImportError as e:
|
|
20
|
+
_check("pyobjc frameworks", False,
|
|
21
|
+
f"pip install pyobjc-framework-Quartz pyobjc-framework-Vision ({e})")
|
|
22
|
+
return 1
|
|
23
|
+
|
|
24
|
+
from ApplicationServices import AXIsProcessTrusted
|
|
25
|
+
ok &= _check(
|
|
26
|
+
"Accessibility permission (taps & keystrokes)", AXIsProcessTrusted(),
|
|
27
|
+
"System Settings > Privacy & Security > Accessibility: enable your terminal")
|
|
28
|
+
|
|
29
|
+
import Quartz as Q
|
|
30
|
+
ok &= _check(
|
|
31
|
+
"Screen Recording permission (seeing the phone)",
|
|
32
|
+
bool(Q.CGPreflightScreenCaptureAccess()),
|
|
33
|
+
"System Settings > Privacy & Security > Screen Recording: enable your terminal")
|
|
34
|
+
|
|
35
|
+
from . import mirror
|
|
36
|
+
ok &= _check(f"{mirror.APP_NAME} installed", Path(mirror.APP_PATH).exists(),
|
|
37
|
+
"requires macOS Sequoia+ with a paired iPhone")
|
|
38
|
+
|
|
39
|
+
running = mirror.running_app() is not None
|
|
40
|
+
_check(f"{mirror.APP_NAME} running", running,
|
|
41
|
+
"will auto-launch on first use โ not fatal")
|
|
42
|
+
|
|
43
|
+
win = mirror.find_window()
|
|
44
|
+
_check("mirroring window found", win is not None,
|
|
45
|
+
"open iPhone Mirroring once manually to pair the phone")
|
|
46
|
+
|
|
47
|
+
if win:
|
|
48
|
+
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f:
|
|
49
|
+
path = f.name
|
|
50
|
+
try:
|
|
51
|
+
subprocess.run(
|
|
52
|
+
["screencapture", "-x", "-o", "-l", str(win["id"]), path],
|
|
53
|
+
check=True)
|
|
54
|
+
size = os.path.getsize(path)
|
|
55
|
+
ok &= _check(f"window capture works ({size} bytes)", size > 20_000,
|
|
56
|
+
"capture is blank โ Screen Recording permission "
|
|
57
|
+
"needs a terminal restart to take effect")
|
|
58
|
+
if size > 20_000:
|
|
59
|
+
from . import ocr
|
|
60
|
+
n = len(ocr.recognize(path, win))
|
|
61
|
+
_check(f"Vision OCR works ({n} text boxes)", True)
|
|
62
|
+
finally:
|
|
63
|
+
os.unlink(path)
|
|
64
|
+
|
|
65
|
+
print("\nall clear" if ok else "\nfix the FAILs above, then re-run")
|
|
66
|
+
print("\nnote: these are the permissions currently known to be required. A "
|
|
67
|
+
"fresh\nmachine may still prompt for more the first time an action "
|
|
68
|
+
"runs โ approve\nthem in System Settings if a step silently does "
|
|
69
|
+
"nothing despite this passing.")
|
|
70
|
+
return 0 if ok else 1
|