macos-computer-use-kit 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- macos_computer_use_kit-0.2.0/.github/workflows/ci.yml +45 -0
- macos_computer_use_kit-0.2.0/.gitignore +22 -0
- macos_computer_use_kit-0.2.0/CHANGELOG.md +60 -0
- macos_computer_use_kit-0.2.0/CONTRIBUTING.md +70 -0
- macos_computer_use_kit-0.2.0/LICENSE +21 -0
- macos_computer_use_kit-0.2.0/PKG-INFO +308 -0
- macos_computer_use_kit-0.2.0/PUBLISHING.md +109 -0
- macos_computer_use_kit-0.2.0/README.md +258 -0
- macos_computer_use_kit-0.2.0/install.sh +127 -0
- macos_computer_use_kit-0.2.0/packages/dsh/LICENSE +21 -0
- macos_computer_use_kit-0.2.0/packages/dsh/README.md +94 -0
- macos_computer_use_kit-0.2.0/packages/dsh/cordis.patch.yml +3 -0
- macos_computer_use_kit-0.2.0/packages/dsh/package-lock.json +305 -0
- macos_computer_use_kit-0.2.0/packages/dsh/package.json +68 -0
- macos_computer_use_kit-0.2.0/packages/dsh/src/index.ts +158 -0
- macos_computer_use_kit-0.2.0/packages/dsh/src/macosCu.ts +83 -0
- macos_computer_use_kit-0.2.0/packages/dsh/tsconfig.json +25 -0
- macos_computer_use_kit-0.2.0/packages/pi/LICENSE +21 -0
- macos_computer_use_kit-0.2.0/packages/pi/README.md +111 -0
- macos_computer_use_kit-0.2.0/packages/pi/extensions/macos-computer-use.ts +595 -0
- macos_computer_use_kit-0.2.0/packages/pi/package-lock.json +3265 -0
- macos_computer_use_kit-0.2.0/packages/pi/package.json +61 -0
- macos_computer_use_kit-0.2.0/packages/pi/skills/macos-computer-use/SKILL.md +186 -0
- macos_computer_use_kit-0.2.0/packages/pi/skills/macos-computer-use/reference/jev-best-practices.md +93 -0
- macos_computer_use_kit-0.2.0/packages/pi/tsconfig.json +21 -0
- macos_computer_use_kit-0.2.0/pyproject.toml +55 -0
- macos_computer_use_kit-0.2.0/skill/SKILL.md +175 -0
- macos_computer_use_kit-0.2.0/skill/reference/jev-best-practices.md +93 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/__init__.py +9 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/__main__.py +4 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/ax.py +363 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/cli.py +204 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/darwin.py +261 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/input_events.py +172 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/jev.py +223 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/overlay.py +148 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/paste.py +111 -0
- macos_computer_use_kit-0.2.0/src/macos_computer_use/shot.py +87 -0
- macos_computer_use_kit-0.2.0/tests/test_policy.py +73 -0
- macos_computer_use_kit-0.2.0/tools/assist.py +14 -0
- macos_computer_use_kit-0.2.0/tools/ax_tool.py +14 -0
- macos_computer_use_kit-0.2.0/tools/guard.py +14 -0
- macos_computer_use_kit-0.2.0/tools/jev_select.py +14 -0
- macos_computer_use_kit-0.2.0/tools/overlay.py +14 -0
- macos_computer_use_kit-0.2.0/tools/shot.py +14 -0
- macos_computer_use_kit-0.2.0/tools/smart_paste.py +14 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
python:
|
|
10
|
+
# Real macOS runner: the CLI is macOS-only, and doctor must not crash in CI
|
|
11
|
+
# where the TCC permissions are absent.
|
|
12
|
+
runs-on: macos-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- name: Install
|
|
19
|
+
run: python -m pip install --upgrade pip && pip install -e ".[dev]"
|
|
20
|
+
- name: Unit tests
|
|
21
|
+
run: pytest -q
|
|
22
|
+
- name: CLI smoke (doctor reports missing permissions instead of crashing)
|
|
23
|
+
run: |
|
|
24
|
+
macos-cu --version
|
|
25
|
+
macos-cu doctor || true
|
|
26
|
+
- name: Byte-compile every module and shim
|
|
27
|
+
run: python -m compileall -q src tools
|
|
28
|
+
|
|
29
|
+
packages:
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
strategy:
|
|
32
|
+
fail-fast: false
|
|
33
|
+
matrix:
|
|
34
|
+
package: [packages/pi, packages/dsh]
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@v4
|
|
37
|
+
- uses: actions/setup-node@v4
|
|
38
|
+
with:
|
|
39
|
+
node-version: "22"
|
|
40
|
+
- name: Install (runs the dsh bundle `prepare` build)
|
|
41
|
+
working-directory: ${{ matrix.package }}
|
|
42
|
+
run: npm ci
|
|
43
|
+
- name: Typecheck
|
|
44
|
+
working-directory: ${{ matrix.package }}
|
|
45
|
+
run: npm run typecheck
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.pyc
|
|
4
|
+
*.egg-info/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
|
|
11
|
+
# Node
|
|
12
|
+
node_modules/
|
|
13
|
+
|
|
14
|
+
# Built dsh bundle output (rebuilt by `prepare` on install/publish)
|
|
15
|
+
packages/*/lib/
|
|
16
|
+
|
|
17
|
+
# Runtime artifacts
|
|
18
|
+
.runtime/
|
|
19
|
+
*.png
|
|
20
|
+
|
|
21
|
+
# macOS
|
|
22
|
+
.DS_Store
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project. The CLI, the pi package, and the dsh bundle
|
|
4
|
+
share a version number.
|
|
5
|
+
|
|
6
|
+
## 0.2.0
|
|
7
|
+
|
|
8
|
+
First release meant for use outside the machine it was built on.
|
|
9
|
+
|
|
10
|
+
**Packaging**
|
|
11
|
+
- Ship an installable Python package (`macos-computer-use-kit`) with a `macos-cu`
|
|
12
|
+
console script. Previously the toolkit was a folder of scripts plus a
|
|
13
|
+
hand-made virtualenv at a fixed absolute path.
|
|
14
|
+
- `tools/*.py` remain as compatibility shims over the same modules.
|
|
15
|
+
- Add `packages/pi` (`pi-package`: skill + 9 tools) and `packages/dsh` (Cordis
|
|
16
|
+
bundle: `cordis.patch.yml` + 5 tools). Both bridge to the CLI with argv arrays
|
|
17
|
+
(`shell: false`) and resolve it from known install locations, so a
|
|
18
|
+
GUI-launched agent works without an interactive shell's `PATH`.
|
|
19
|
+
- `install.sh` now provisions a venv, a launcher, the agent skill, and the
|
|
20
|
+
optional Jev key, and picks whichever package index answers faster.
|
|
21
|
+
|
|
22
|
+
**Portability fixes**
|
|
23
|
+
- Remove the hardcoded display scale (`0.9333 = 1372/1470`); `--shot-scale` is
|
|
24
|
+
explicit opt-in and there is no fake default.
|
|
25
|
+
- Stop hardcoding the overlay path, the snapshot cache directory, and the
|
|
26
|
+
`/tmp/shot.png` capture path.
|
|
27
|
+
- Add `unsupported_platform` / `missing_dependency` JSON errors instead of a
|
|
28
|
+
pyobjc traceback or a shell error.
|
|
29
|
+
- Deterministic app resolution: exact bundle id, exact localized name,
|
|
30
|
+
case-insensitive exact, then substring; frontmost apps win ties. The old
|
|
31
|
+
behaviour took the first substring match, which could target the wrong app.
|
|
32
|
+
- Prefer `NSPasteboardTypeString`, falling back to the legacy
|
|
33
|
+
`NSStringPboardType`.
|
|
34
|
+
|
|
35
|
+
**Behaviour fixes**
|
|
36
|
+
- `ax find` now honours `--max` (it used to return every match, flooding
|
|
37
|
+
context).
|
|
38
|
+
- `input windows --pid` actually filters by pid.
|
|
39
|
+
- `doctor` reports platform, permissions, display geometry, dependencies, and
|
|
40
|
+
Jev setup as JSON, with actionable hints.
|
|
41
|
+
- The Jev guard policy is a pure `decide()` function; the model version is
|
|
42
|
+
pinnable with `TYPESAFE_MODEL`.
|
|
43
|
+
|
|
44
|
+
**Documentation**
|
|
45
|
+
- English-first README and skill, with a concise Chinese section; machine- and
|
|
46
|
+
app-specific walkthroughs replaced by generic examples.
|
|
47
|
+
- `PUBLISHING.md` with the npm/PyPI release steps and catalog-listing criteria.
|
|
48
|
+
- Correct the dependency list (`pyobjc-framework-AppKit` does not exist; AppKit
|
|
49
|
+
ships in `pyobjc-framework-Cocoa`).
|
|
50
|
+
|
|
51
|
+
**Tests and CI**
|
|
52
|
+
- Unit tests for the guard policy and the window target signature.
|
|
53
|
+
- GitHub Actions: tests and a compile check on macOS, typecheck for both npm
|
|
54
|
+
packages.
|
|
55
|
+
|
|
56
|
+
## 0.1.0
|
|
57
|
+
|
|
58
|
+
Initial extraction of the toolkit from three implementations (Codex CUA, ZCode
|
|
59
|
+
Computer Use, Grok Bot), with `ax_tool.py`, `assist.py`, `smart_paste.py`,
|
|
60
|
+
`shot.py`, `overlay.py`, `guard.py`, `jev_select.py`, and an agent skill.
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Bug reports, platform findings, and pull requests are welcome. This toolkit
|
|
4
|
+
drives real desktops, so evidence beats opinion: paste the JSON a command
|
|
5
|
+
returned, the macOS version, and the app you were driving.
|
|
6
|
+
|
|
7
|
+
## Layout
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
src/macos_computer_use/ the CLI (single source of truth)
|
|
11
|
+
tools/*.py compatibility shims over the same modules
|
|
12
|
+
skill/ agent skill + Jev reference
|
|
13
|
+
packages/pi/ pi package (skill + tools; sources, no build)
|
|
14
|
+
packages/dsh/ DeepSeek Harness bundle (TypeScript -> lib/)
|
|
15
|
+
tests/ unit tests for pure logic
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Develop
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
python -m venv .venv && . .venv/bin/activate
|
|
22
|
+
pip install -e ".[dev]"
|
|
23
|
+
pytest -q
|
|
24
|
+
macos-cu doctor
|
|
25
|
+
|
|
26
|
+
(cd packages/pi && npm ci && npm run typecheck)
|
|
27
|
+
(cd packages/dsh && npm ci && npm run typecheck && npm run build)
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Conventions
|
|
31
|
+
|
|
32
|
+
- **JSON on stdout, diagnostics on stderr.** Every command prints one JSON
|
|
33
|
+
object (or JSON lines) so an agent can consume it. Keep the exit codes stable:
|
|
34
|
+
`0` ok, `2` usage/permission/platform, `3` not found or upstream error,
|
|
35
|
+
`4` capture failed, `5` target changed.
|
|
36
|
+
- **Never claim success you did not verify.** Emitted-the-event (`action_sent`)
|
|
37
|
+
and the-UI-changed (`verified`) are separate facts, and every fallback hint
|
|
38
|
+
must follow from evidence.
|
|
39
|
+
- **No machine-specific defaults.** No absolute paths, no assumed display scale,
|
|
40
|
+
no assumed app names. If a value depends on the machine, detect it or require
|
|
41
|
+
it explicitly.
|
|
42
|
+
- **Pure logic stays pure.** Policy functions such as `jev.decide()` take values
|
|
43
|
+
and return a decision, so they can be unit-tested without a desktop or a
|
|
44
|
+
network.
|
|
45
|
+
- **Both bridges stay bridges.** The pi and dsh packages call the CLI with argv
|
|
46
|
+
arrays and return its output; they must not grow a second implementation, and
|
|
47
|
+
they must never use a shell string.
|
|
48
|
+
- **Keep the skill honest.** If a capability changes, update `skill/SKILL.md` —
|
|
49
|
+
it is what agents actually read. `packages/pi/skills/...` mirrors it.
|
|
50
|
+
|
|
51
|
+
## Adding a command
|
|
52
|
+
|
|
53
|
+
1. Implement it in the matching `src/macos_computer_use/*.py` module.
|
|
54
|
+
2. Wire it into `build_parser()` in `cli.py` and into the `doctor` hints if it
|
|
55
|
+
needs a permission.
|
|
56
|
+
3. Add or extend a test when the logic is pure.
|
|
57
|
+
4. Update `README.md`, `skill/SKILL.md`, and (for a new desktop action) both
|
|
58
|
+
bridge packages' tool lists.
|
|
59
|
+
|
|
60
|
+
## Reporting a computer-use bug
|
|
61
|
+
|
|
62
|
+
Include: the exact command, the JSON it returned, the app and its bundle id, the
|
|
63
|
+
macOS version, and whether the app was frontmost. If AX targeting failed, say
|
|
64
|
+
whether `verified` was false and whether the element is custom-drawn — that is
|
|
65
|
+
the difference between a bug here and an app that does not implement
|
|
66
|
+
accessibility.
|
|
67
|
+
|
|
68
|
+
## License
|
|
69
|
+
|
|
70
|
+
By contributing you agree your work is released under the MIT License.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sur-Cai
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: macos-computer-use-kit
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: AX-first computer-use toolkit for AI agents on macOS: accessibility-tree targeting, process-scoped input, clipboard-safe paste, action read-back verification, visual feedback, and optional Jev semantic guards.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Sur-Cai/macos-computer-use-kit
|
|
6
|
+
Project-URL: Repository, https://github.com/Sur-Cai/macos-computer-use-kit
|
|
7
|
+
Project-URL: Issues, https://github.com/Sur-Cai/macos-computer-use-kit/issues
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 Sur-Cai
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Keywords: accessibility,agent-tools,ai-agent,automation,computer-use,dsh-plugin,macos,opencode,pi-package
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Environment :: MacOS X :: Cocoa
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
36
|
+
Classifier: Programming Language :: Python :: 3
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
41
|
+
Classifier: Topic :: Software Development :: User Interfaces
|
|
42
|
+
Requires-Python: >=3.10
|
|
43
|
+
Requires-Dist: pillow>=10.0
|
|
44
|
+
Requires-Dist: pyobjc-framework-applicationservices>=10.0; sys_platform == 'darwin'
|
|
45
|
+
Requires-Dist: pyobjc-framework-cocoa>=10.0; sys_platform == 'darwin'
|
|
46
|
+
Requires-Dist: pyobjc-framework-quartz>=10.0; sys_platform == 'darwin'
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
49
|
+
Description-Content-Type: text/markdown
|
|
50
|
+
|
|
51
|
+
# macos-computer-use-kit
|
|
52
|
+
|
|
53
|
+
**AX-first computer use for AI agents on macOS.** Instead of screenshot → eyeball
|
|
54
|
+
coordinates → click and hope, read the accessibility tree, get each element's
|
|
55
|
+
semantics and exact geometry, act on it, then verify the action actually changed
|
|
56
|
+
the UI.
|
|
57
|
+
|
|
58
|
+
A small, composable toolkit: accessibility-tree targeting, process- and
|
|
59
|
+
window-scoped input, clipboard-safe pasting, action read-back verification,
|
|
60
|
+
blank-frame detection, visual feedback, and optional Jev (TypeSafe System One)
|
|
61
|
+
semantic guards.
|
|
62
|
+
|
|
63
|
+
Works with any agent that can run a shell command, and ships first-class
|
|
64
|
+
packages for [pi](#pi) and [DeepSeek Harness](#deepseek-harness-dsh).
|
|
65
|
+
|
|
66
|
+
## Why
|
|
67
|
+
|
|
68
|
+
These mechanisms were distilled from three mature implementations rather than
|
|
69
|
+
invented from scratch:
|
|
70
|
+
|
|
71
|
+
| Source | Mechanism absorbed |
|
|
72
|
+
| --- | --- |
|
|
73
|
+
| Codex CUA (`@oai/cua` / Sky service) | AX state + element index, `setValue`, batch actions, event delivery via `CGEventPostToPid` |
|
|
74
|
+
| ZCode Computer Use | `*_to_window` window-scoped input, `target_changed` validation, clipboard-safe paste pipeline, `screenshot_blank` |
|
|
75
|
+
| Grok Bot (`CUGrokBotService`) | snapshots with stable element ids + text budget + drill-down, action read-back, coordinate fallback on failure |
|
|
76
|
+
|
|
77
|
+
## Install
|
|
78
|
+
|
|
79
|
+
macOS 12+, Python 3.10+.
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install macos-computer-use-kit # or: pipx install macos-computer-use-kit
|
|
83
|
+
macos-cu doctor # check permissions, displays, dependencies
|
|
84
|
+
|
|
85
|
+
# from a checkout (editable install + agent skill)
|
|
86
|
+
git clone https://github.com/Sur-Cai/macos-computer-use-kit && cd macos-computer-use-kit
|
|
87
|
+
./install.sh
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Grant both permissions to the process that runs the agent (your terminal, or the
|
|
91
|
+
agent app). `macos-cu doctor` reports what is missing and where to enable it:
|
|
92
|
+
|
|
93
|
+
- **Accessibility** — AX reads, `AXPress`, `setValue`, posted events
|
|
94
|
+
- **Screen Recording** — `shot` (without it every capture is black)
|
|
95
|
+
|
|
96
|
+
## Quickstart
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
# semantic targeting: exact geometry, zero visual reasoning
|
|
100
|
+
macos-cu ax find --app com.apple.finder --role AXButton --title Size
|
|
101
|
+
macos-cu ax tree --app com.apple.finder --depth 16 --max 200
|
|
102
|
+
|
|
103
|
+
# token-efficient snapshot with stable ids, trimmed to a budget
|
|
104
|
+
macos-cu ax snapshot --app com.apple.finder --budget 1200 --file /tmp/ax.json
|
|
105
|
+
macos-cu ax resolve --file /tmp/ax.json --id 0.1.0.6.0.0.0.0.5.8
|
|
106
|
+
|
|
107
|
+
# native AX action + read-back verification
|
|
108
|
+
macos-cu ax press --app com.apple.finder --role AXButton --title Size
|
|
109
|
+
# {"verified":true,"state_changed":true,...}
|
|
110
|
+
|
|
111
|
+
# window-scoped input: the user's cursor never moves, target is validated
|
|
112
|
+
macos-cu input windows --app "Google Chrome"
|
|
113
|
+
macos-cu input click --window-id 12345 --x 171 --y 28 --show
|
|
114
|
+
macos-cu input click --window-id 12345 --x 171 --y 28 --expect "37040:12345:642:244:824:640"
|
|
115
|
+
# mismatch -> {"ok":false,"reason":"target_changed"} and exit code 5
|
|
116
|
+
|
|
117
|
+
# clipboard-safe paste (saves and restores the user's clipboard)
|
|
118
|
+
macos-cu paste --app com.google.Chrome --text "你好" --mode pid
|
|
119
|
+
|
|
120
|
+
# screenshot with blank-frame detection
|
|
121
|
+
macos-cu shot capture --app "Google Chrome" --out /tmp/shot.png
|
|
122
|
+
macos-cu shot check --file /tmp/shot.png
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## CLI
|
|
126
|
+
|
|
127
|
+
One binary, JSON output, stable exit codes (`0` ok, `2` usage/permission,
|
|
128
|
+
`3` not found, `4` capture failed, `5` target changed).
|
|
129
|
+
|
|
130
|
+
| Group | Commands |
|
|
131
|
+
| --- | --- |
|
|
132
|
+
| `macos-cu ax` | `tree`, `find`, `click-info`, `snapshot`, `resolve`, `press`, `setvalue` |
|
|
133
|
+
| `macos-cu input` | `windows`, `cursor`, `pid`, `click`, `key`, `scroll`, `move` |
|
|
134
|
+
| `macos-cu paste` | clipboard-safe paste (`--mode pid\|hid`, `--keep`) |
|
|
135
|
+
| `macos-cu shot` | `capture`, `check`, `windows` |
|
|
136
|
+
| `macos-cu overlay` | `show`, `clear` |
|
|
137
|
+
| `macos-cu jev` | `guard`, `select` (optional, JSON on stdin) |
|
|
138
|
+
| `macos-cu doctor` | permissions, displays, dependencies, Jev setup |
|
|
139
|
+
|
|
140
|
+
### Coordinate spaces
|
|
141
|
+
|
|
142
|
+
| Space | Source | Used by |
|
|
143
|
+
| --- | --- | --- |
|
|
144
|
+
| `screen[x, y]` | AX/CoreGraphics points, origin at the primary display's top-left | `input --x --y`, `overlay` |
|
|
145
|
+
| window-relative | element point − window origin | `input click --window-id N --x --y` |
|
|
146
|
+
| `shot[x, y]` | `center_screen × --shot-scale` | only for harnesses whose screenshots are scaled differently from screen points; there is deliberately no default |
|
|
147
|
+
|
|
148
|
+
Secondary displays placed left of or above the primary produce **negative**
|
|
149
|
+
coordinates. That is normal. `macos-cu doctor` prints the layout.
|
|
150
|
+
|
|
151
|
+
## Agent integrations
|
|
152
|
+
|
|
153
|
+
| Harness | What you get | Install |
|
|
154
|
+
| --- | --- | --- |
|
|
155
|
+
| any agent with a shell | the full CLI | `pip install macos-computer-use-kit` |
|
|
156
|
+
| [opencode](https://opencode.ai) | skill `macos-computer-use` (auto-discovered) | `./install.sh` |
|
|
157
|
+
| [pi](https://pi.dev) | skill + 9 native tools | `pi install npm:pi-macos-computer-use` |
|
|
158
|
+
| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | plugin bundle, 5 tools | `dsh plugin --profile <name> add dsh-macos-computer-use` |
|
|
159
|
+
|
|
160
|
+
<a name="pi"></a>
|
|
161
|
+
### pi package
|
|
162
|
+
|
|
163
|
+
`packages/pi` — `pi-macos-computer-use` (npm, `pi-package` keyword). Registers
|
|
164
|
+
`macos_cu_doctor`, `macos_ax_find`, `macos_ax_press`, `macos_input_windows`,
|
|
165
|
+
`macos_input_click`, `macos_input_key`, `macos_paste`, `macos_shot`,
|
|
166
|
+
`macos_jev_guard`. Every tool shells out with an argv array (`shell: false`), so
|
|
167
|
+
model-supplied text can never reach a shell.
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
pi install npm:pi-macos-computer-use
|
|
171
|
+
pi -e ./packages/pi # try it for one run without installing
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
App launchers do not inherit your interactive shell's `PATH`. If the CLI is
|
|
175
|
+
installed but pi cannot find it, set `MACOS_CU_BIN=/abs/path/to/macos-cu` and
|
|
176
|
+
restart pi (the dsh plugin honours the same variable).
|
|
177
|
+
|
|
178
|
+
<a name="deepseek-harness-dsh"></a>
|
|
179
|
+
### DeepSeek Harness plugin
|
|
180
|
+
|
|
181
|
+
`packages/dsh` — `dsh-macos-computer-use`, a Cordis bundle
|
|
182
|
+
(`dsh.bundle.patch` → `cordis.patch.yml`). Registers `macos_cu_doctor`,
|
|
183
|
+
`macos_ax_find`, `macos_ax_press`, `macos_input_click`, `macos_shot`.
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
dsh plugin --profile demo add dsh-macos-computer-use
|
|
187
|
+
dsh --profile demo --dump-config # verify the layer before booting
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
It deliberately does **not** claim the exclusive `ctx.computerUse` provider slot:
|
|
191
|
+
it adds tools rather than owning desktop operations, so it cannot block the
|
|
192
|
+
in-box Cua Driver provider. See `packages/dsh/README.md`.
|
|
193
|
+
|
|
194
|
+
### opencode skill
|
|
195
|
+
|
|
196
|
+
`./install.sh` installs `skill/SKILL.md` to
|
|
197
|
+
`~/.config/opencode/skills/macos-computer-use/`, where opencode discovers it
|
|
198
|
+
automatically.
|
|
199
|
+
|
|
200
|
+
## The four capabilities that matter
|
|
201
|
+
|
|
202
|
+
**1. Window-scoped input with target validation.** Events are posted straight to
|
|
203
|
+
the target process (`CGEventPostToPid`), so the physical cursor never moves and
|
|
204
|
+
the user can keep working. `--expect pid:wid:x:y:w:h` refuses to act when the
|
|
205
|
+
window moved or lost focus since you looked at it.
|
|
206
|
+
|
|
207
|
+
**2. Native AX actions with read-back verification.** `press`/`setvalue` compare
|
|
208
|
+
the window's visible-text fingerprint and focused element before and after, and
|
|
209
|
+
report `verified` separately from `action_sent`. When AX cannot act (custom-drawn
|
|
210
|
+
UI), the result carries a `hint` telling you to fall back to a coordinate click
|
|
211
|
+
or a clipboard paste — you find out from evidence, not from guessing.
|
|
212
|
+
|
|
213
|
+
**3. Clipboard safety.** The user's clipboard is saved before and restored after.
|
|
214
|
+
Takeover and non-consumption are reported explicitly, so pasting CJK text never
|
|
215
|
+
silently destroys what the user had copied.
|
|
216
|
+
|
|
217
|
+
**4. Never reason on a blank frame.** `shot` classifies captures as
|
|
218
|
+
`ok` / `all_black` / `all_white` / `uniform` with a hint, so a missing permission
|
|
219
|
+
or an occluded window is reported instead of hallucinated UI state.
|
|
220
|
+
|
|
221
|
+
## Design principles
|
|
222
|
+
|
|
223
|
+
1. **AX-first.** Semantic + exact geometry beats visual inference. Screenshots
|
|
224
|
+
verify; they do not target.
|
|
225
|
+
2. **`action_sent` ≠ `verified`.** Keep "we emitted the event" and "the UI
|
|
226
|
+
changed" as separate facts.
|
|
227
|
+
3. **The clipboard is a shared resource.** Save, detect interference, restore.
|
|
228
|
+
4. **Targets must be explicit and checkable.** Window input carries a signature;
|
|
229
|
+
a mismatch is `target_changed`, not a misclick.
|
|
230
|
+
5. **Small models judge, code decides.** Jev returns calibrated probabilities;
|
|
231
|
+
thresholds and side effects stay in code. Cost ladder: deterministic code
|
|
232
|
+
(µs) < Jev (~1 s) < visual reasoning (seconds to tens of seconds).
|
|
233
|
+
6. **Make it visible.** Action points draw a ring, so the user is never watching
|
|
234
|
+
a black box.
|
|
235
|
+
|
|
236
|
+
## Jev semantic guards (optional)
|
|
237
|
+
|
|
238
|
+
The only part that needs a key. Everything else works without it.
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
echo '{"task":"send the report to Alice",
|
|
242
|
+
"expected":{"recipient":"Alice","message":"Q3 numbers"},
|
|
243
|
+
"observed":{"chat_title":"Bob","input_text":"Q3 numbers"}}' | macos-cu jev guard
|
|
244
|
+
# {"answers":{"right_target":0.02,"input_ok":0.98,"blocker":"wrong_target"},
|
|
245
|
+
# "decision":"switch_target"}
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
One request fans out independent judgments and **code** applies the policy:
|
|
249
|
+
proceed only when `blocker=none` and both probabilities clear the threshold; the
|
|
250
|
+
model may only suggest the two safe recoveries (`switch_target`, `retype_input`);
|
|
251
|
+
anything else asks the user. `macos-cu jev select` picks one candidate element
|
|
252
|
+
with a `none` escape hatch and a confidence gate.
|
|
253
|
+
|
|
254
|
+
Key: `TYPESAFE_API_KEY` or `~/.config/typesafe/api_key`
|
|
255
|
+
(<https://console.typesafe.ai/keys>). Pin `TYPESAFE_MODEL` for automation;
|
|
256
|
+
`jev-latest` is the friendly default. Question-design guidance lives in
|
|
257
|
+
[`skill/reference/jev-best-practices.md`](skill/reference/jev-best-practices.md).
|
|
258
|
+
|
|
259
|
+
## Known limitations
|
|
260
|
+
|
|
261
|
+
- macOS only (AX, CGEvent, ScreenCaptureKit are macOS APIs).
|
|
262
|
+
- Custom-drawn UIs (some Electron apps, games, chat apps) expose shallow or
|
|
263
|
+
uncooperative AX trees. Fall back to screenshots **after** read-back fails,
|
|
264
|
+
not before.
|
|
265
|
+
- Process-targeted key events are accepted by most apps but not all: browsers
|
|
266
|
+
usually accept background keystrokes; some chat apps require the app to be
|
|
267
|
+
frontmost for typing and pasting.
|
|
268
|
+
- AX coordinate scale is display-dependent. `center_shot` is opt-in via
|
|
269
|
+
`--shot-scale` for exactly this reason.
|
|
270
|
+
- The user may be using the machine at the same time. Concurrent automation is
|
|
271
|
+
risky; one extra verification before an irreversible action is cheap.
|
|
272
|
+
|
|
273
|
+
## Repository layout
|
|
274
|
+
|
|
275
|
+
```
|
|
276
|
+
src/macos_computer_use/ the CLI implementation (pip-installable)
|
|
277
|
+
tools/*.py compatibility shims -> the same modules
|
|
278
|
+
skill/ agent skill (SKILL.md + Jev reference)
|
|
279
|
+
packages/pi/ pi package (skill + native tools)
|
|
280
|
+
packages/dsh/ DeepSeek Harness plugin bundle
|
|
281
|
+
install.sh local installer (venv + CLI + skill + Jev key)
|
|
282
|
+
tests/ unit tests for the safety-relevant logic
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
Contributions welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for the
|
|
286
|
+
conventions (JSON on stdout, stable exit codes, no machine-specific defaults).
|
|
287
|
+
Release steps and catalog-listing criteria live in
|
|
288
|
+
[PUBLISHING.md](PUBLISHING.md); notable changes are in
|
|
289
|
+
[CHANGELOG.md](CHANGELOG.md).
|
|
290
|
+
|
|
291
|
+
## 中文说明
|
|
292
|
+
|
|
293
|
+
给 AI agent 用的 macOS 电脑控制工具箱:**AX 语义定位**(不靠截图目测坐标)、
|
|
294
|
+
进程/窗口级输入(物理光标不动)、剪贴板安全粘贴、动作回读校验、空白帧检测、
|
|
295
|
+
可视反馈,以及可选的 Jev 语义护栏。
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
pip install macos-computer-use-kit
|
|
299
|
+
macos-cu doctor # 检查辅助功能 / 屏幕录制权限、显示器、依赖、Jev
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
Agent 集成:`./install.sh`(opencode skill)、`pi install npm:pi-macos-computer-use`(pi)、
|
|
303
|
+
`dsh plugin --profile <名> add dsh-macos-computer-use`(DeepSeek Harness)。
|
|
304
|
+
完整流程与避坑见 [`skill/SKILL.md`](skill/SKILL.md)。
|
|
305
|
+
|
|
306
|
+
## License
|
|
307
|
+
|
|
308
|
+
MIT
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Publishing this toolkit
|
|
2
|
+
|
|
3
|
+
Three artifacts can be published from this repository, all versioned in lockstep
|
|
4
|
+
with the CLI (`0.2.0` today):
|
|
5
|
+
|
|
6
|
+
| Artifact | Registry | Command |
|
|
7
|
+
| --- | --- | --- |
|
|
8
|
+
| `macos-computer-use-kit` | PyPI | `python -m build && twine upload dist/*` |
|
|
9
|
+
| `pi-macos-computer-use` | npm | `cd packages/pi && npm publish` |
|
|
10
|
+
| `dsh-macos-computer-use` | npm | `cd packages/dsh && npm publish` |
|
|
11
|
+
|
|
12
|
+
## Before the first publish
|
|
13
|
+
|
|
14
|
+
1. **Never commit a key.** The Jev key lives in `~/.config/typesafe/api_key` and
|
|
15
|
+
the environment. Check `git status` and `git diff --cached` before pushing;
|
|
16
|
+
`.gitignore` already excludes `node_modules/`, build output, and `*.png`.
|
|
17
|
+
2. **Bump all three versions together.** They are independent manifests:
|
|
18
|
+
`pyproject.toml`, `packages/pi/package.json`, `packages/dsh/package.json`.
|
|
19
|
+
`packages/pi/skills/macos-computer-use/SKILL.md` is a copy of `skill/SKILL.md`
|
|
20
|
+
— refresh it (`cp skill/SKILL.md packages/pi/skills/macos-computer-use/SKILL.md`)
|
|
21
|
+
and re-run `./install.sh` to refresh the opencode copy.
|
|
22
|
+
3. **Add GitHub topics** for discoverability (Repository → About → Topics):
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
dsh-plugin pi-package macos computer-use accessibility
|
|
26
|
+
ai-agents agent-skills gui-automation opencode
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`dsh-plugin` is how <https://github.com/topics/dsh-plugin> indexes plugins;
|
|
30
|
+
the `pi-package` npm keyword is what <https://pi.dev/packages> indexes.
|
|
31
|
+
|
|
32
|
+
## PyPI (the CLI)
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
python -m pip install --upgrade build twine
|
|
36
|
+
python -m build # sdist + wheel in dist/
|
|
37
|
+
twine check dist/*
|
|
38
|
+
twine upload dist/* # needs a PyPI API token
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Verify the install path in a clean venv before tagging:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python -m venv /tmp/verify && /tmp/verify/bin/pip install dist/*.whl
|
|
45
|
+
/tmp/verify/bin/macos-cu doctor
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## npm: the pi package
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
cd packages/pi
|
|
52
|
+
npm run typecheck
|
|
53
|
+
npm pack --dry-run # confirm the tarball contents
|
|
54
|
+
npm publish # unscoped; use --access public inside a scope
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The `pi` manifest (`extensions`, `skills`) plus the `pi-package` keyword are what
|
|
58
|
+
make the catalog pick it up. The extension is loaded through pi's TypeScript
|
|
59
|
+
loader, so it ships sources — no build step, and `peerDependencies` stay `"*"`.
|
|
60
|
+
|
|
61
|
+
## npm: the dsh bundle
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
cd packages/dsh
|
|
65
|
+
npm run build # emits lib/ ; `prepare` runs this on git installs
|
|
66
|
+
npm pack --dry-run
|
|
67
|
+
npm publish
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The bundle contract is `dsh.bundle.patch` → `cordis.patch.yml`. Because git
|
|
71
|
+
installs fetch **sources**, the `prepare` script must stay self-contained (no
|
|
72
|
+
monorepo project references); publishing a tarball or npm package ships the
|
|
73
|
+
built `lib/` and needs no build permission from the user.
|
|
74
|
+
|
|
75
|
+
Users install with:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
dsh plugin --profile <name> add dsh-macos-computer-use # from npm (recommended)
|
|
79
|
+
dsh plugin --profile <name> add ./dsh-macos-computer-use-0.2.0.tgz # from a packed tarball
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The npm and tarball forms ship the prebuilt `lib/`, so no build permission is
|
|
83
|
+
needed. Installing straight from git fetches sources and runs `prepare`, which
|
|
84
|
+
pnpm blocks until the user allowlists it — and because the bundle lives in
|
|
85
|
+
`packages/dsh/` of this monorepo, the git form is only convenient for a repo
|
|
86
|
+
that root-mounts the package. Prefer npm or the tarball for distribution.
|
|
87
|
+
|
|
88
|
+
## Catalog listings
|
|
89
|
+
|
|
90
|
+
- **pi**: publishing to npm is enough — the gallery at <https://pi.dev/packages>
|
|
91
|
+
lists packages tagged `pi-package`. Optionally add a `pi.image` or
|
|
92
|
+
`pi.video` preview to `packages/pi/package.json` for a richer card.
|
|
93
|
+
- **dsh**: add the `dsh-plugin` GitHub topic, then open a PR against
|
|
94
|
+
[awesome-dsh-plugin](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin)
|
|
95
|
+
(their bar: installs with `dsh plugin add`, does what its one-line description
|
|
96
|
+
says, right category, maintained). There is also
|
|
97
|
+
[dsh-market](https://github.com/dsh-market/dsh-market) for in-app discovery.
|
|
98
|
+
|
|
99
|
+
## Release checklist
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
./install.sh # local sanity: CLI + skill
|
|
103
|
+
macos-cu doctor # permissions still granted after reinstalling
|
|
104
|
+
(cd packages/pi && npm run typecheck)
|
|
105
|
+
(cd packages/dsh && npm run typecheck && npm run build)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Then tag (`git tag v0.2.0 && git push --tags`) and publish the artifacts you
|
|
109
|
+
intend to ship.
|