macos-computer-use-kit 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. macos_computer_use_kit-0.2.0/.github/workflows/ci.yml +45 -0
  2. macos_computer_use_kit-0.2.0/.gitignore +22 -0
  3. macos_computer_use_kit-0.2.0/CHANGELOG.md +60 -0
  4. macos_computer_use_kit-0.2.0/CONTRIBUTING.md +70 -0
  5. macos_computer_use_kit-0.2.0/LICENSE +21 -0
  6. macos_computer_use_kit-0.2.0/PKG-INFO +308 -0
  7. macos_computer_use_kit-0.2.0/PUBLISHING.md +109 -0
  8. macos_computer_use_kit-0.2.0/README.md +258 -0
  9. macos_computer_use_kit-0.2.0/install.sh +127 -0
  10. macos_computer_use_kit-0.2.0/packages/dsh/LICENSE +21 -0
  11. macos_computer_use_kit-0.2.0/packages/dsh/README.md +94 -0
  12. macos_computer_use_kit-0.2.0/packages/dsh/cordis.patch.yml +3 -0
  13. macos_computer_use_kit-0.2.0/packages/dsh/package-lock.json +305 -0
  14. macos_computer_use_kit-0.2.0/packages/dsh/package.json +68 -0
  15. macos_computer_use_kit-0.2.0/packages/dsh/src/index.ts +158 -0
  16. macos_computer_use_kit-0.2.0/packages/dsh/src/macosCu.ts +83 -0
  17. macos_computer_use_kit-0.2.0/packages/dsh/tsconfig.json +25 -0
  18. macos_computer_use_kit-0.2.0/packages/pi/LICENSE +21 -0
  19. macos_computer_use_kit-0.2.0/packages/pi/README.md +111 -0
  20. macos_computer_use_kit-0.2.0/packages/pi/extensions/macos-computer-use.ts +595 -0
  21. macos_computer_use_kit-0.2.0/packages/pi/package-lock.json +3265 -0
  22. macos_computer_use_kit-0.2.0/packages/pi/package.json +61 -0
  23. macos_computer_use_kit-0.2.0/packages/pi/skills/macos-computer-use/SKILL.md +186 -0
  24. macos_computer_use_kit-0.2.0/packages/pi/skills/macos-computer-use/reference/jev-best-practices.md +93 -0
  25. macos_computer_use_kit-0.2.0/packages/pi/tsconfig.json +21 -0
  26. macos_computer_use_kit-0.2.0/pyproject.toml +55 -0
  27. macos_computer_use_kit-0.2.0/skill/SKILL.md +175 -0
  28. macos_computer_use_kit-0.2.0/skill/reference/jev-best-practices.md +93 -0
  29. macos_computer_use_kit-0.2.0/src/macos_computer_use/__init__.py +9 -0
  30. macos_computer_use_kit-0.2.0/src/macos_computer_use/__main__.py +4 -0
  31. macos_computer_use_kit-0.2.0/src/macos_computer_use/ax.py +363 -0
  32. macos_computer_use_kit-0.2.0/src/macos_computer_use/cli.py +204 -0
  33. macos_computer_use_kit-0.2.0/src/macos_computer_use/darwin.py +261 -0
  34. macos_computer_use_kit-0.2.0/src/macos_computer_use/input_events.py +172 -0
  35. macos_computer_use_kit-0.2.0/src/macos_computer_use/jev.py +223 -0
  36. macos_computer_use_kit-0.2.0/src/macos_computer_use/overlay.py +148 -0
  37. macos_computer_use_kit-0.2.0/src/macos_computer_use/paste.py +111 -0
  38. macos_computer_use_kit-0.2.0/src/macos_computer_use/shot.py +87 -0
  39. macos_computer_use_kit-0.2.0/tests/test_policy.py +73 -0
  40. macos_computer_use_kit-0.2.0/tools/assist.py +14 -0
  41. macos_computer_use_kit-0.2.0/tools/ax_tool.py +14 -0
  42. macos_computer_use_kit-0.2.0/tools/guard.py +14 -0
  43. macos_computer_use_kit-0.2.0/tools/jev_select.py +14 -0
  44. macos_computer_use_kit-0.2.0/tools/overlay.py +14 -0
  45. macos_computer_use_kit-0.2.0/tools/shot.py +14 -0
  46. macos_computer_use_kit-0.2.0/tools/smart_paste.py +14 -0
@@ -0,0 +1,45 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ python:
10
+ # Real macOS runner: the CLI is macOS-only, and doctor must not crash in CI
11
+ # where the TCC permissions are absent.
12
+ runs-on: macos-latest
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+ - uses: actions/setup-python@v5
16
+ with:
17
+ python-version: "3.12"
18
+ - name: Install
19
+ run: python -m pip install --upgrade pip && pip install -e ".[dev]"
20
+ - name: Unit tests
21
+ run: pytest -q
22
+ - name: CLI smoke (doctor reports missing permissions instead of crashing)
23
+ run: |
24
+ macos-cu --version
25
+ macos-cu doctor || true
26
+ - name: Byte-compile every module and shim
27
+ run: python -m compileall -q src tools
28
+
29
+ packages:
30
+ runs-on: ubuntu-latest
31
+ strategy:
32
+ fail-fast: false
33
+ matrix:
34
+ package: [packages/pi, packages/dsh]
35
+ steps:
36
+ - uses: actions/checkout@v4
37
+ - uses: actions/setup-node@v4
38
+ with:
39
+ node-version: "22"
40
+ - name: Install (runs the dsh bundle `prepare` build)
41
+ working-directory: ${{ matrix.package }}
42
+ run: npm ci
43
+ - name: Typecheck
44
+ working-directory: ${{ matrix.package }}
45
+ run: npm run typecheck
@@ -0,0 +1,22 @@
1
+ # Python
2
+ __pycache__/
3
+ *.pyc
4
+ *.egg-info/
5
+ dist/
6
+ build/
7
+ .pytest_cache/
8
+ .venv/
9
+ venv/
10
+
11
+ # Node
12
+ node_modules/
13
+
14
+ # Built dsh bundle output (rebuilt by `prepare` on install/publish)
15
+ packages/*/lib/
16
+
17
+ # Runtime artifacts
18
+ .runtime/
19
+ *.png
20
+
21
+ # macOS
22
+ .DS_Store
@@ -0,0 +1,60 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project. The CLI, the pi package, and the dsh bundle
4
+ share a version number.
5
+
6
+ ## 0.2.0
7
+
8
+ First release meant for use outside the machine it was built on.
9
+
10
+ **Packaging**
11
+ - Ship an installable Python package (`macos-computer-use-kit`) with a `macos-cu`
12
+ console script. Previously the toolkit was a folder of scripts plus a
13
+ hand-made virtualenv at a fixed absolute path.
14
+ - `tools/*.py` remain as compatibility shims over the same modules.
15
+ - Add `packages/pi` (`pi-package`: skill + 9 tools) and `packages/dsh` (Cordis
16
+ bundle: `cordis.patch.yml` + 5 tools). Both bridge to the CLI with argv arrays
17
+ (`shell: false`) and resolve it from known install locations, so a
18
+ GUI-launched agent works without an interactive shell's `PATH`.
19
+ - `install.sh` now provisions a venv, a launcher, the agent skill, and the
20
+ optional Jev key, and picks whichever package index answers faster.
21
+
22
+ **Portability fixes**
23
+ - Remove the hardcoded display scale (`0.9333 = 1372/1470`); `--shot-scale` is
24
+ explicit opt-in and there is no fake default.
25
+ - Stop hardcoding the overlay path, the snapshot cache directory, and the
26
+ `/tmp/shot.png` capture path.
27
+ - Add `unsupported_platform` / `missing_dependency` JSON errors instead of a
28
+ pyobjc traceback or a shell error.
29
+ - Deterministic app resolution: exact bundle id, exact localized name,
30
+ case-insensitive exact, then substring; frontmost apps win ties. The old
31
+ behaviour took the first substring match, which could target the wrong app.
32
+ - Prefer `NSPasteboardTypeString`, falling back to the legacy
33
+ `NSStringPboardType`.
34
+
35
+ **Behaviour fixes**
36
+ - `ax find` now honours `--max` (it used to return every match, flooding
37
+ context).
38
+ - `input windows --pid` actually filters by pid.
39
+ - `doctor` reports platform, permissions, display geometry, dependencies, and
40
+ Jev setup as JSON, with actionable hints.
41
+ - The Jev guard policy is a pure `decide()` function; the model version is
42
+ pinnable with `TYPESAFE_MODEL`.
43
+
44
+ **Documentation**
45
+ - English-first README and skill, with a concise Chinese section; machine- and
46
+ app-specific walkthroughs replaced by generic examples.
47
+ - `PUBLISHING.md` with the npm/PyPI release steps and catalog-listing criteria.
48
+ - Correct the dependency list (`pyobjc-framework-AppKit` does not exist; AppKit
49
+ ships in `pyobjc-framework-Cocoa`).
50
+
51
+ **Tests and CI**
52
+ - Unit tests for the guard policy and the window target signature.
53
+ - GitHub Actions: tests and a compile check on macOS, typecheck for both npm
54
+ packages.
55
+
56
+ ## 0.1.0
57
+
58
+ Initial extraction of the toolkit from three implementations (Codex CUA, ZCode
59
+ Computer Use, Grok Bot), with `ax_tool.py`, `assist.py`, `smart_paste.py`,
60
+ `shot.py`, `overlay.py`, `guard.py`, `jev_select.py`, and an agent skill.
@@ -0,0 +1,70 @@
1
+ # Contributing
2
+
3
+ Bug reports, platform findings, and pull requests are welcome. This toolkit
4
+ drives real desktops, so evidence beats opinion: paste the JSON a command
5
+ returned, the macOS version, and the app you were driving.
6
+
7
+ ## Layout
8
+
9
+ ```
10
+ src/macos_computer_use/ the CLI (single source of truth)
11
+ tools/*.py compatibility shims over the same modules
12
+ skill/ agent skill + Jev reference
13
+ packages/pi/ pi package (skill + tools; sources, no build)
14
+ packages/dsh/ DeepSeek Harness bundle (TypeScript -> lib/)
15
+ tests/ unit tests for pure logic
16
+ ```
17
+
18
+ ## Develop
19
+
20
+ ```bash
21
+ python -m venv .venv && . .venv/bin/activate
22
+ pip install -e ".[dev]"
23
+ pytest -q
24
+ macos-cu doctor
25
+
26
+ (cd packages/pi && npm ci && npm run typecheck)
27
+ (cd packages/dsh && npm ci && npm run typecheck && npm run build)
28
+ ```
29
+
30
+ ## Conventions
31
+
32
+ - **JSON on stdout, diagnostics on stderr.** Every command prints one JSON
33
+ object (or JSON lines) so an agent can consume it. Keep the exit codes stable:
34
+ `0` ok, `2` usage/permission/platform, `3` not found or upstream error,
35
+ `4` capture failed, `5` target changed.
36
+ - **Never claim success you did not verify.** Emitted-the-event (`action_sent`)
37
+ and the-UI-changed (`verified`) are separate facts, and every fallback hint
38
+ must follow from evidence.
39
+ - **No machine-specific defaults.** No absolute paths, no assumed display scale,
40
+ no assumed app names. If a value depends on the machine, detect it or require
41
+ it explicitly.
42
+ - **Pure logic stays pure.** Policy functions such as `jev.decide()` take values
43
+ and return a decision, so they can be unit-tested without a desktop or a
44
+ network.
45
+ - **Both bridges stay bridges.** The pi and dsh packages call the CLI with argv
46
+ arrays and return its output; they must not grow a second implementation, and
47
+ they must never use a shell string.
48
+ - **Keep the skill honest.** If a capability changes, update `skill/SKILL.md` —
49
+ it is what agents actually read. `packages/pi/skills/...` mirrors it.
50
+
51
+ ## Adding a command
52
+
53
+ 1. Implement it in the matching `src/macos_computer_use/*.py` module.
54
+ 2. Wire it into `build_parser()` in `cli.py` and into the `doctor` hints if it
55
+ needs a permission.
56
+ 3. Add or extend a test when the logic is pure.
57
+ 4. Update `README.md`, `skill/SKILL.md`, and (for a new desktop action) both
58
+ bridge packages' tool lists.
59
+
60
+ ## Reporting a computer-use bug
61
+
62
+ Include: the exact command, the JSON it returned, the app and its bundle id, the
63
+ macOS version, and whether the app was frontmost. If AX targeting failed, say
64
+ whether `verified` was false and whether the element is custom-drawn — that is
65
+ the difference between a bug here and an app that does not implement
66
+ accessibility.
67
+
68
+ ## License
69
+
70
+ By contributing you agree your work is released under the MIT License.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sur-Cai
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,308 @@
1
+ Metadata-Version: 2.5
2
+ Name: macos-computer-use-kit
3
+ Version: 0.2.0
4
+ Summary: AX-first computer-use toolkit for AI agents on macOS: accessibility-tree targeting, process-scoped input, clipboard-safe paste, action read-back verification, visual feedback, and optional Jev semantic guards.
5
+ Project-URL: Homepage, https://github.com/Sur-Cai/macos-computer-use-kit
6
+ Project-URL: Repository, https://github.com/Sur-Cai/macos-computer-use-kit
7
+ Project-URL: Issues, https://github.com/Sur-Cai/macos-computer-use-kit/issues
8
+ License: MIT License
9
+
10
+ Copyright (c) 2026 Sur-Cai
11
+
12
+ Permission is hereby granted, free of charge, to any person obtaining a copy
13
+ of this software and associated documentation files (the "Software"), to deal
14
+ in the Software without restriction, including without limitation the rights
15
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16
+ copies of the Software, and to permit persons to whom the Software is
17
+ furnished to do so, subject to the following conditions:
18
+
19
+ The above copyright notice and this permission notice shall be included in all
20
+ copies or substantial portions of the Software.
21
+
22
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28
+ SOFTWARE.
29
+ License-File: LICENSE
30
+ Keywords: accessibility,agent-tools,ai-agent,automation,computer-use,dsh-plugin,macos,opencode,pi-package
31
+ Classifier: Development Status :: 4 - Beta
32
+ Classifier: Environment :: MacOS X :: Cocoa
33
+ Classifier: Intended Audience :: Developers
34
+ Classifier: License :: OSI Approved :: MIT License
35
+ Classifier: Operating System :: MacOS :: MacOS X
36
+ Classifier: Programming Language :: Python :: 3
37
+ Classifier: Programming Language :: Python :: 3.10
38
+ Classifier: Programming Language :: Python :: 3.11
39
+ Classifier: Programming Language :: Python :: 3.12
40
+ Classifier: Programming Language :: Python :: 3.13
41
+ Classifier: Topic :: Software Development :: User Interfaces
42
+ Requires-Python: >=3.10
43
+ Requires-Dist: pillow>=10.0
44
+ Requires-Dist: pyobjc-framework-applicationservices>=10.0; sys_platform == 'darwin'
45
+ Requires-Dist: pyobjc-framework-cocoa>=10.0; sys_platform == 'darwin'
46
+ Requires-Dist: pyobjc-framework-quartz>=10.0; sys_platform == 'darwin'
47
+ Provides-Extra: dev
48
+ Requires-Dist: pytest>=8.0; extra == 'dev'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # macos-computer-use-kit
52
+
53
+ **AX-first computer use for AI agents on macOS.** Instead of screenshot → eyeball
54
+ coordinates → click and hope, read the accessibility tree, get each element's
55
+ semantics and exact geometry, act on it, then verify the action actually changed
56
+ the UI.
57
+
58
+ A small, composable toolkit: accessibility-tree targeting, process- and
59
+ window-scoped input, clipboard-safe pasting, action read-back verification,
60
+ blank-frame detection, visual feedback, and optional Jev (TypeSafe System One)
61
+ semantic guards.
62
+
63
+ Works with any agent that can run a shell command, and ships first-class
64
+ packages for [pi](#pi) and [DeepSeek Harness](#deepseek-harness-dsh).
65
+
66
+ ## Why
67
+
68
+ These mechanisms were distilled from three mature implementations rather than
69
+ invented from scratch:
70
+
71
+ | Source | Mechanism absorbed |
72
+ | --- | --- |
73
+ | Codex CUA (`@oai/cua` / Sky service) | AX state + element index, `setValue`, batch actions, event delivery via `CGEventPostToPid` |
74
+ | ZCode Computer Use | `*_to_window` window-scoped input, `target_changed` validation, clipboard-safe paste pipeline, `screenshot_blank` |
75
+ | Grok Bot (`CUGrokBotService`) | snapshots with stable element ids + text budget + drill-down, action read-back, coordinate fallback on failure |
76
+
77
+ ## Install
78
+
79
+ macOS 12+, Python 3.10+.
80
+
81
+ ```bash
82
+ pip install macos-computer-use-kit # or: pipx install macos-computer-use-kit
83
+ macos-cu doctor # check permissions, displays, dependencies
84
+
85
+ # from a checkout (editable install + agent skill)
86
+ git clone https://github.com/Sur-Cai/macos-computer-use-kit && cd macos-computer-use-kit
87
+ ./install.sh
88
+ ```
89
+
90
+ Grant both permissions to the process that runs the agent (your terminal, or the
91
+ agent app). `macos-cu doctor` reports what is missing and where to enable it:
92
+
93
+ - **Accessibility** — AX reads, `AXPress`, `setValue`, posted events
94
+ - **Screen Recording** — `shot` (without it every capture is black)
95
+
96
+ ## Quickstart
97
+
98
+ ```bash
99
+ # semantic targeting: exact geometry, zero visual reasoning
100
+ macos-cu ax find --app com.apple.finder --role AXButton --title Size
101
+ macos-cu ax tree --app com.apple.finder --depth 16 --max 200
102
+
103
+ # token-efficient snapshot with stable ids, trimmed to a budget
104
+ macos-cu ax snapshot --app com.apple.finder --budget 1200 --file /tmp/ax.json
105
+ macos-cu ax resolve --file /tmp/ax.json --id 0.1.0.6.0.0.0.0.5.8
106
+
107
+ # native AX action + read-back verification
108
+ macos-cu ax press --app com.apple.finder --role AXButton --title Size
109
+ # {"verified":true,"state_changed":true,...}
110
+
111
+ # window-scoped input: the user's cursor never moves, target is validated
112
+ macos-cu input windows --app "Google Chrome"
113
+ macos-cu input click --window-id 12345 --x 171 --y 28 --show
114
+ macos-cu input click --window-id 12345 --x 171 --y 28 --expect "37040:12345:642:244:824:640"
115
+ # mismatch -> {"ok":false,"reason":"target_changed"} and exit code 5
116
+
117
+ # clipboard-safe paste (saves and restores the user's clipboard)
118
+ macos-cu paste --app com.google.Chrome --text "你好" --mode pid
119
+
120
+ # screenshot with blank-frame detection
121
+ macos-cu shot capture --app "Google Chrome" --out /tmp/shot.png
122
+ macos-cu shot check --file /tmp/shot.png
123
+ ```
124
+
125
+ ## CLI
126
+
127
+ One binary, JSON output, stable exit codes (`0` ok, `2` usage/permission,
128
+ `3` not found, `4` capture failed, `5` target changed).
129
+
130
+ | Group | Commands |
131
+ | --- | --- |
132
+ | `macos-cu ax` | `tree`, `find`, `click-info`, `snapshot`, `resolve`, `press`, `setvalue` |
133
+ | `macos-cu input` | `windows`, `cursor`, `pid`, `click`, `key`, `scroll`, `move` |
134
+ | `macos-cu paste` | clipboard-safe paste (`--mode pid\|hid`, `--keep`) |
135
+ | `macos-cu shot` | `capture`, `check`, `windows` |
136
+ | `macos-cu overlay` | `show`, `clear` |
137
+ | `macos-cu jev` | `guard`, `select` (optional, JSON on stdin) |
138
+ | `macos-cu doctor` | permissions, displays, dependencies, Jev setup |
139
+
140
+ ### Coordinate spaces
141
+
142
+ | Space | Source | Used by |
143
+ | --- | --- | --- |
144
+ | `screen[x, y]` | AX/CoreGraphics points, origin at the primary display's top-left | `input --x --y`, `overlay` |
145
+ | window-relative | element point − window origin | `input click --window-id N --x --y` |
146
+ | `shot[x, y]` | `center_screen × --shot-scale` | only for harnesses whose screenshots are scaled differently from screen points; there is deliberately no default |
147
+
148
+ Secondary displays placed left of or above the primary produce **negative**
149
+ coordinates. That is normal. `macos-cu doctor` prints the layout.
150
+
151
+ ## Agent integrations
152
+
153
+ | Harness | What you get | Install |
154
+ | --- | --- | --- |
155
+ | any agent with a shell | the full CLI | `pip install macos-computer-use-kit` |
156
+ | [opencode](https://opencode.ai) | skill `macos-computer-use` (auto-discovered) | `./install.sh` |
157
+ | [pi](https://pi.dev) | skill + 9 native tools | `pi install npm:pi-macos-computer-use` |
158
+ | [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | plugin bundle, 5 tools | `dsh plugin --profile <name> add dsh-macos-computer-use` |
159
+
160
+ <a name="pi"></a>
161
+ ### pi package
162
+
163
+ `packages/pi` — `pi-macos-computer-use` (npm, `pi-package` keyword). Registers
164
+ `macos_cu_doctor`, `macos_ax_find`, `macos_ax_press`, `macos_input_windows`,
165
+ `macos_input_click`, `macos_input_key`, `macos_paste`, `macos_shot`,
166
+ `macos_jev_guard`. Every tool shells out with an argv array (`shell: false`), so
167
+ model-supplied text can never reach a shell.
168
+
169
+ ```bash
170
+ pi install npm:pi-macos-computer-use
171
+ pi -e ./packages/pi # try it for one run without installing
172
+ ```
173
+
174
+ App launchers do not inherit your interactive shell's `PATH`. If the CLI is
175
+ installed but pi cannot find it, set `MACOS_CU_BIN=/abs/path/to/macos-cu` and
176
+ restart pi (the dsh plugin honours the same variable).
177
+
178
+ <a name="deepseek-harness-dsh"></a>
179
+ ### DeepSeek Harness plugin
180
+
181
+ `packages/dsh` — `dsh-macos-computer-use`, a Cordis bundle
182
+ (`dsh.bundle.patch` → `cordis.patch.yml`). Registers `macos_cu_doctor`,
183
+ `macos_ax_find`, `macos_ax_press`, `macos_input_click`, `macos_shot`.
184
+
185
+ ```bash
186
+ dsh plugin --profile demo add dsh-macos-computer-use
187
+ dsh --profile demo --dump-config # verify the layer before booting
188
+ ```
189
+
190
+ It deliberately does **not** claim the exclusive `ctx.computerUse` provider slot:
191
+ it adds tools rather than owning desktop operations, so it cannot block the
192
+ in-box Cua Driver provider. See `packages/dsh/README.md`.
193
+
194
+ ### opencode skill
195
+
196
+ `./install.sh` installs `skill/SKILL.md` to
197
+ `~/.config/opencode/skills/macos-computer-use/`, where opencode discovers it
198
+ automatically.
199
+
200
+ ## The four capabilities that matter
201
+
202
+ **1. Window-scoped input with target validation.** Events are posted straight to
203
+ the target process (`CGEventPostToPid`), so the physical cursor never moves and
204
+ the user can keep working. `--expect pid:wid:x:y:w:h` refuses to act when the
205
+ window moved or lost focus since you looked at it.
206
+
207
+ **2. Native AX actions with read-back verification.** `press`/`setvalue` compare
208
+ the window's visible-text fingerprint and focused element before and after, and
209
+ report `verified` separately from `action_sent`. When AX cannot act (custom-drawn
210
+ UI), the result carries a `hint` telling you to fall back to a coordinate click
211
+ or a clipboard paste — you find out from evidence, not from guessing.
212
+
213
+ **3. Clipboard safety.** The user's clipboard is saved before and restored after.
214
+ Takeover and non-consumption are reported explicitly, so pasting CJK text never
215
+ silently destroys what the user had copied.
216
+
217
+ **4. Never reason on a blank frame.** `shot` classifies captures as
218
+ `ok` / `all_black` / `all_white` / `uniform` with a hint, so a missing permission
219
+ or an occluded window is reported instead of hallucinated UI state.
220
+
221
+ ## Design principles
222
+
223
+ 1. **AX-first.** Semantic + exact geometry beats visual inference. Screenshots
224
+ verify; they do not target.
225
+ 2. **`action_sent` ≠ `verified`.** Keep "we emitted the event" and "the UI
226
+ changed" as separate facts.
227
+ 3. **The clipboard is a shared resource.** Save, detect interference, restore.
228
+ 4. **Targets must be explicit and checkable.** Window input carries a signature;
229
+ a mismatch is `target_changed`, not a misclick.
230
+ 5. **Small models judge, code decides.** Jev returns calibrated probabilities;
231
+ thresholds and side effects stay in code. Cost ladder: deterministic code
232
+ (µs) < Jev (~1 s) < visual reasoning (seconds to tens of seconds).
233
+ 6. **Make it visible.** Action points draw a ring, so the user is never watching
234
+ a black box.
235
+
236
+ ## Jev semantic guards (optional)
237
+
238
+ The only part that needs a key. Everything else works without it.
239
+
240
+ ```bash
241
+ echo '{"task":"send the report to Alice",
242
+ "expected":{"recipient":"Alice","message":"Q3 numbers"},
243
+ "observed":{"chat_title":"Bob","input_text":"Q3 numbers"}}' | macos-cu jev guard
244
+ # {"answers":{"right_target":0.02,"input_ok":0.98,"blocker":"wrong_target"},
245
+ # "decision":"switch_target"}
246
+ ```
247
+
248
+ One request fans out independent judgments and **code** applies the policy:
249
+ proceed only when `blocker=none` and both probabilities clear the threshold; the
250
+ model may only suggest the two safe recoveries (`switch_target`, `retype_input`);
251
+ anything else asks the user. `macos-cu jev select` picks one candidate element
252
+ with a `none` escape hatch and a confidence gate.
253
+
254
+ Key: `TYPESAFE_API_KEY` or `~/.config/typesafe/api_key`
255
+ (<https://console.typesafe.ai/keys>). Pin `TYPESAFE_MODEL` for automation;
256
+ `jev-latest` is the friendly default. Question-design guidance lives in
257
+ [`skill/reference/jev-best-practices.md`](skill/reference/jev-best-practices.md).
258
+
259
+ ## Known limitations
260
+
261
+ - macOS only (AX, CGEvent, ScreenCaptureKit are macOS APIs).
262
+ - Custom-drawn UIs (some Electron apps, games, chat apps) expose shallow or
263
+ uncooperative AX trees. Fall back to screenshots **after** read-back fails,
264
+ not before.
265
+ - Process-targeted key events are accepted by most apps but not all: browsers
266
+ usually accept background keystrokes; some chat apps require the app to be
267
+ frontmost for typing and pasting.
268
+ - AX coordinate scale is display-dependent. `center_shot` is opt-in via
269
+ `--shot-scale` for exactly this reason.
270
+ - The user may be using the machine at the same time. Concurrent automation is
271
+ risky; one extra verification before an irreversible action is cheap.
272
+
273
+ ## Repository layout
274
+
275
+ ```
276
+ src/macos_computer_use/ the CLI implementation (pip-installable)
277
+ tools/*.py compatibility shims -> the same modules
278
+ skill/ agent skill (SKILL.md + Jev reference)
279
+ packages/pi/ pi package (skill + native tools)
280
+ packages/dsh/ DeepSeek Harness plugin bundle
281
+ install.sh local installer (venv + CLI + skill + Jev key)
282
+ tests/ unit tests for the safety-relevant logic
283
+ ```
284
+
285
+ Contributions welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for the
286
+ conventions (JSON on stdout, stable exit codes, no machine-specific defaults).
287
+ Release steps and catalog-listing criteria live in
288
+ [PUBLISHING.md](PUBLISHING.md); notable changes are in
289
+ [CHANGELOG.md](CHANGELOG.md).
290
+
291
+ ## 中文说明
292
+
293
+ 给 AI agent 用的 macOS 电脑控制工具箱:**AX 语义定位**(不靠截图目测坐标)、
294
+ 进程/窗口级输入(物理光标不动)、剪贴板安全粘贴、动作回读校验、空白帧检测、
295
+ 可视反馈,以及可选的 Jev 语义护栏。
296
+
297
+ ```bash
298
+ pip install macos-computer-use-kit
299
+ macos-cu doctor # 检查辅助功能 / 屏幕录制权限、显示器、依赖、Jev
300
+ ```
301
+
302
+ Agent 集成:`./install.sh`(opencode skill)、`pi install npm:pi-macos-computer-use`(pi)、
303
+ `dsh plugin --profile <名> add dsh-macos-computer-use`(DeepSeek Harness)。
304
+ 完整流程与避坑见 [`skill/SKILL.md`](skill/SKILL.md)。
305
+
306
+ ## License
307
+
308
+ MIT
@@ -0,0 +1,109 @@
1
+ # Publishing this toolkit
2
+
3
+ Three artifacts can be published from this repository, all versioned in lockstep
4
+ with the CLI (`0.2.0` today):
5
+
6
+ | Artifact | Registry | Command |
7
+ | --- | --- | --- |
8
+ | `macos-computer-use-kit` | PyPI | `python -m build && twine upload dist/*` |
9
+ | `pi-macos-computer-use` | npm | `cd packages/pi && npm publish` |
10
+ | `dsh-macos-computer-use` | npm | `cd packages/dsh && npm publish` |
11
+
12
+ ## Before the first publish
13
+
14
+ 1. **Never commit a key.** The Jev key lives in `~/.config/typesafe/api_key` and
15
+ the environment. Check `git status` and `git diff --cached` before pushing;
16
+ `.gitignore` already excludes `node_modules/`, build output, and `*.png`.
17
+ 2. **Bump all three versions together.** They are independent manifests:
18
+ `pyproject.toml`, `packages/pi/package.json`, `packages/dsh/package.json`.
19
+ `packages/pi/skills/macos-computer-use/SKILL.md` is a copy of `skill/SKILL.md`
20
+ — refresh it (`cp skill/SKILL.md packages/pi/skills/macos-computer-use/SKILL.md`)
21
+ and re-run `./install.sh` to refresh the opencode copy.
22
+ 3. **Add GitHub topics** for discoverability (Repository → About → Topics):
23
+
24
+ ```
25
+ dsh-plugin pi-package macos computer-use accessibility
26
+ ai-agents agent-skills gui-automation opencode
27
+ ```
28
+
29
+ `dsh-plugin` is how <https://github.com/topics/dsh-plugin> indexes plugins;
30
+ the `pi-package` npm keyword is what <https://pi.dev/packages> indexes.
31
+
32
+ ## PyPI (the CLI)
33
+
34
+ ```bash
35
+ python -m pip install --upgrade build twine
36
+ python -m build # sdist + wheel in dist/
37
+ twine check dist/*
38
+ twine upload dist/* # needs a PyPI API token
39
+ ```
40
+
41
+ Verify the install path in a clean venv before tagging:
42
+
43
+ ```bash
44
+ python -m venv /tmp/verify && /tmp/verify/bin/pip install dist/*.whl
45
+ /tmp/verify/bin/macos-cu doctor
46
+ ```
47
+
48
+ ## npm: the pi package
49
+
50
+ ```bash
51
+ cd packages/pi
52
+ npm run typecheck
53
+ npm pack --dry-run # confirm the tarball contents
54
+ npm publish # unscoped; use --access public inside a scope
55
+ ```
56
+
57
+ The `pi` manifest (`extensions`, `skills`) plus the `pi-package` keyword are what
58
+ make the catalog pick it up. The extension is loaded through pi's TypeScript
59
+ loader, so it ships sources — no build step, and `peerDependencies` stay `"*"`.
60
+
61
+ ## npm: the dsh bundle
62
+
63
+ ```bash
64
+ cd packages/dsh
65
+ npm run build # emits lib/ ; `prepare` runs this on git installs
66
+ npm pack --dry-run
67
+ npm publish
68
+ ```
69
+
70
+ The bundle contract is `dsh.bundle.patch` → `cordis.patch.yml`. Because git
71
+ installs fetch **sources**, the `prepare` script must stay self-contained (no
72
+ monorepo project references); publishing a tarball or npm package ships the
73
+ built `lib/` and needs no build permission from the user.
74
+
75
+ Users install with:
76
+
77
+ ```bash
78
+ dsh plugin --profile <name> add dsh-macos-computer-use # from npm (recommended)
79
+ dsh plugin --profile <name> add ./dsh-macos-computer-use-0.2.0.tgz # from a packed tarball
80
+ ```
81
+
82
+ The npm and tarball forms ship the prebuilt `lib/`, so no build permission is
83
+ needed. Installing straight from git fetches sources and runs `prepare`, which
84
+ pnpm blocks until the user allowlists it — and because the bundle lives in
85
+ `packages/dsh/` of this monorepo, the git form is only convenient for a repo
86
+ that root-mounts the package. Prefer npm or the tarball for distribution.
87
+
88
+ ## Catalog listings
89
+
90
+ - **pi**: publishing to npm is enough — the gallery at <https://pi.dev/packages>
91
+ lists packages tagged `pi-package`. Optionally add a `pi.image` or
92
+ `pi.video` preview to `packages/pi/package.json` for a richer card.
93
+ - **dsh**: add the `dsh-plugin` GitHub topic, then open a PR against
94
+ [awesome-dsh-plugin](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin)
95
+ (their bar: installs with `dsh plugin add`, does what its one-line description
96
+ says, right category, maintained). There is also
97
+ [dsh-market](https://github.com/dsh-market/dsh-market) for in-app discovery.
98
+
99
+ ## Release checklist
100
+
101
+ ```bash
102
+ ./install.sh # local sanity: CLI + skill
103
+ macos-cu doctor # permissions still granted after reinstalling
104
+ (cd packages/pi && npm run typecheck)
105
+ (cd packages/dsh && npm run typecheck && npm run build)
106
+ ```
107
+
108
+ Then tag (`git tag v0.2.0 && git push --tags`) and publish the artifacts you
109
+ intend to ship.