macos-computer-use-kit 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/CHANGELOG.md +16 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/PKG-INFO +62 -21
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/PUBLISHING.md +33 -6
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/README.md +59 -19
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/README.md +2 -1
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/package.json +6 -3
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/src/index.ts +39 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/src/macosCu.ts +6 -2
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/package.json +5 -2
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/pyproject.toml +6 -1
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/__init__.py +1 -1
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/.github/workflows/ci.yml +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/.gitignore +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/CONTRIBUTING.md +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/LICENSE +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/install.sh +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/LICENSE +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/cordis.patch.yml +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/package-lock.json +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/tsconfig.json +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/LICENSE +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/README.md +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/extensions/macos-computer-use.ts +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/package-lock.json +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/skills/macos-computer-use/SKILL.md +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/skills/macos-computer-use/reference/jev-best-practices.md +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/pi/tsconfig.json +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/skill/SKILL.md +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/skill/reference/jev-best-practices.md +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/__main__.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/ax.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/cli.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/darwin.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/input_events.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/jev.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/overlay.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/paste.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/shot.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tests/test_policy.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/assist.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/ax_tool.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/guard.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/jev_select.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/overlay.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/shot.py +0 -0
- {macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/tools/smart_paste.py +0 -0
|
@@ -3,6 +3,22 @@
|
|
|
3
3
|
All notable changes to this project. The CLI, the pi package, and the dsh bundle
|
|
4
4
|
share a version number.
|
|
5
5
|
|
|
6
|
+
## 0.2.2
|
|
7
|
+
|
|
8
|
+
Discoverability and parity for the optional Jev (TypeSafe System One) guards.
|
|
9
|
+
|
|
10
|
+
- **The dsh bundle now exposes `macos_jev_guard`** — the same optional semantic
|
|
11
|
+
guard the pi package already had — so both integrations carry the capability
|
|
12
|
+
they advertise (6 tools instead of 5). The bridge gained stdin support, which
|
|
13
|
+
is how `macos-cu jev` receives its JSON request.
|
|
14
|
+
- npm keywords gain `jev`, `typesafe-ai`, `system-one-models`; PyPI keywords gain
|
|
15
|
+
`jev`, `typesafe-ai`, `system-one`, `llm-guardrails`. Keywords are immutable
|
|
16
|
+
per version, which is why this needed a release rather than an edit.
|
|
17
|
+
- Declare the `Programming Language :: Python :: 3.14` classifier. The wheel has
|
|
18
|
+
installed and run on 3.14 since 0.2.0; only the metadata was behind.
|
|
19
|
+
|
|
20
|
+
No behaviour changes outside the new tool.
|
|
21
|
+
|
|
6
22
|
## 0.2.1
|
|
7
23
|
|
|
8
24
|
- Publish the Python package metadata with an SPDX license expression
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: macos-computer-use-kit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: AX-first computer-use toolkit for AI agents on macOS: accessibility-tree targeting, process-scoped input, clipboard-safe paste, action read-back verification, visual feedback, and optional Jev semantic guards.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Sur-Cai/macos-computer-use-kit
|
|
6
6
|
Project-URL: Repository, https://github.com/Sur-Cai/macos-computer-use-kit
|
|
7
7
|
Project-URL: Issues, https://github.com/Sur-Cai/macos-computer-use-kit/issues
|
|
8
8
|
License-Expression: MIT
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Keywords: accessibility,agent-tools,ai-agent,automation,computer-use,dsh-plugin,macos,opencode,pi-package
|
|
10
|
+
Keywords: accessibility,agent-tools,ai-agent,automation,computer-use,dsh-plugin,jev,llm-guardrails,macos,opencode,pi-package,system-one,typesafe-ai
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: Environment :: MacOS X :: Cocoa
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
22
|
Classifier: Topic :: Software Development :: User Interfaces
|
|
22
23
|
Requires-Python: >=3.10
|
|
23
24
|
Requires-Dist: pillow>=10.0
|
|
@@ -30,6 +31,13 @@ Description-Content-Type: text/markdown
|
|
|
30
31
|
|
|
31
32
|
# macos-computer-use-kit
|
|
32
33
|
|
|
34
|
+
[](https://pypi.org/project/macos-computer-use-kit/)
|
|
35
|
+
[](https://pypi.org/project/macos-computer-use-kit/)
|
|
36
|
+
[](https://www.npmjs.com/package/pi-macos-computer-use)
|
|
37
|
+
[](https://www.npmjs.com/package/dsh-macos-computer-use)
|
|
38
|
+
[](https://github.com/Sur-Cai/macos-computer-use-kit/actions/workflows/ci.yml)
|
|
39
|
+
[](LICENSE)
|
|
40
|
+
|
|
33
41
|
**AX-first computer use for AI agents on macOS.** Instead of screenshot → eyeball
|
|
34
42
|
coordinates → click and hope, read the accessibility tree, get each element's
|
|
35
43
|
semantics and exact geometry, act on it, then verify the action actually changed
|
|
@@ -40,6 +48,14 @@ window-scoped input, clipboard-safe pasting, action read-back verification,
|
|
|
40
48
|
blank-frame detection, visual feedback, and optional Jev (TypeSafe System One)
|
|
41
49
|
semantic guards.
|
|
42
50
|
|
|
51
|
+
**Jev is the semantic guard layer.** A small model — no reasoning tokens, no
|
|
52
|
+
prose — returns calibrated judgments right before an irreversible action: *is
|
|
53
|
+
this still the intended recipient? does the field hold the intended text? what,
|
|
54
|
+
if anything, blocks the action?* The code, not the model, decides whether to
|
|
55
|
+
proceed, and the model may only ever suggest the two recoveries that cannot send
|
|
56
|
+
or submit anything. Probabilities in, decisions out. See
|
|
57
|
+
[Jev semantic guards](#jev-semantic-guards-optional).
|
|
58
|
+
|
|
43
59
|
Works with any agent that can run a shell command, and ships first-class
|
|
44
60
|
packages for [pi](#pi) and [DeepSeek Harness](#deepseek-harness-dsh).
|
|
45
61
|
|
|
@@ -59,10 +75,19 @@ invented from scratch:
|
|
|
59
75
|
macOS 12+, Python 3.10+.
|
|
60
76
|
|
|
61
77
|
```bash
|
|
62
|
-
|
|
63
|
-
macos-
|
|
78
|
+
# 1) the CLI — every integration below drives this, and it works on its own
|
|
79
|
+
pip install macos-computer-use-kit # or: pipx install macos-computer-use-kit
|
|
80
|
+
macos-cu doctor # permissions, displays, dependencies
|
|
81
|
+
|
|
82
|
+
# 2) your agent integration (optional — pick one)
|
|
83
|
+
pi install npm:pi-macos-computer-use # pi
|
|
84
|
+
dsh plugin --profile <name> add dsh-macos-computer-use # DeepSeek Harness
|
|
85
|
+
```
|
|
64
86
|
|
|
65
|
-
|
|
87
|
+
From a checkout — this is also the opencode integration (editable CLI, the
|
|
88
|
+
`macos-computer-use` skill, a `macos-cu` launcher, and the optional Jev key):
|
|
89
|
+
|
|
90
|
+
```bash
|
|
66
91
|
git clone https://github.com/Sur-Cai/macos-computer-use-kit && cd macos-computer-use-kit
|
|
67
92
|
./install.sh
|
|
68
93
|
```
|
|
@@ -132,23 +157,30 @@ coordinates. That is normal. `macos-cu doctor` prints the layout.
|
|
|
132
157
|
|
|
133
158
|
| Harness | What you get | Install |
|
|
134
159
|
| --- | --- | --- |
|
|
135
|
-
| any agent with a shell | the full CLI | `pip install macos-computer-use-kit` |
|
|
160
|
+
| any agent with a shell | the full CLI ([PyPI](https://pypi.org/project/macos-computer-use-kit/)) | `pip install macos-computer-use-kit` |
|
|
136
161
|
| [opencode](https://opencode.ai) | skill `macos-computer-use` (auto-discovered) | `./install.sh` |
|
|
137
|
-
| [pi](https://pi.dev) | skill + 9 native tools | `pi install npm:pi-macos-computer-use` |
|
|
138
|
-
| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | plugin bundle,
|
|
162
|
+
| [pi](https://pi.dev) | skill + 9 native tools ([npm](https://www.npmjs.com/package/pi-macos-computer-use), [catalog](https://pi.dev/packages/pi-macos-computer-use)) | `pi install npm:pi-macos-computer-use` |
|
|
163
|
+
| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | plugin bundle, 6 tools ([npm](https://www.npmjs.com/package/dsh-macos-computer-use)) | `dsh plugin --profile <name> add dsh-macos-computer-use` |
|
|
164
|
+
|
|
165
|
+
Every integration is a thin bridge over the same CLI, so install the CLI first
|
|
166
|
+
(`pip install macos-computer-use-kit`). The three published artifacts — the PyPI
|
|
167
|
+
CLI, the pi package, and the dsh bundle — are versioned and released together;
|
|
168
|
+
the badges at the top of this file show the current release.
|
|
139
169
|
|
|
140
170
|
<a name="pi"></a>
|
|
141
171
|
### pi package
|
|
142
172
|
|
|
143
|
-
`
|
|
173
|
+
[`pi-macos-computer-use`](https://www.npmjs.com/package/pi-macos-computer-use) on
|
|
174
|
+
npm (`pi-package` keyword), sources in `packages/pi`. Registers
|
|
144
175
|
`macos_cu_doctor`, `macos_ax_find`, `macos_ax_press`, `macos_input_windows`,
|
|
145
176
|
`macos_input_click`, `macos_input_key`, `macos_paste`, `macos_shot`,
|
|
146
177
|
`macos_jev_guard`. Every tool shells out with an argv array (`shell: false`), so
|
|
147
178
|
model-supplied text can never reach a shell.
|
|
148
179
|
|
|
149
180
|
```bash
|
|
150
|
-
|
|
151
|
-
pi
|
|
181
|
+
pip install macos-computer-use-kit # the CLI the tools call
|
|
182
|
+
pi install npm:pi-macos-computer-use # the integration
|
|
183
|
+
pi -e ./packages/pi # or try a checkout for one run, without installing
|
|
152
184
|
```
|
|
153
185
|
|
|
154
186
|
App launchers do not inherit your interactive shell's `PATH`. If the CLI is
|
|
@@ -158,13 +190,16 @@ restart pi (the dsh plugin honours the same variable).
|
|
|
158
190
|
<a name="deepseek-harness-dsh"></a>
|
|
159
191
|
### DeepSeek Harness plugin
|
|
160
192
|
|
|
161
|
-
`
|
|
193
|
+
[`dsh-macos-computer-use`](https://www.npmjs.com/package/dsh-macos-computer-use)
|
|
194
|
+
on npm, sources in `packages/dsh` — a Cordis bundle
|
|
162
195
|
(`dsh.bundle.patch` → `cordis.patch.yml`). Registers `macos_cu_doctor`,
|
|
163
|
-
`macos_ax_find`, `macos_ax_press`, `macos_input_click`, `macos_shot
|
|
196
|
+
`macos_ax_find`, `macos_ax_press`, `macos_input_click`, `macos_shot`,
|
|
197
|
+
`macos_jev_guard` (the same optional Jev guard the pi package exposes).
|
|
164
198
|
|
|
165
199
|
```bash
|
|
166
|
-
|
|
167
|
-
dsh --profile demo
|
|
200
|
+
pip install macos-computer-use-kit # the CLI the tools call
|
|
201
|
+
dsh plugin --profile demo add dsh-macos-computer-use # the plugin
|
|
202
|
+
dsh --profile demo --dump-config # verify the layer before booting
|
|
168
203
|
```
|
|
169
204
|
|
|
170
205
|
It deliberately does **not** claim the exclusive `ctx.computerUse` provider slot:
|
|
@@ -175,7 +210,7 @@ in-box Cua Driver provider. See `packages/dsh/README.md`.
|
|
|
175
210
|
|
|
176
211
|
`./install.sh` installs `skill/SKILL.md` to
|
|
177
212
|
`~/.config/opencode/skills/macos-computer-use/`, where opencode discovers it
|
|
178
|
-
automatically
|
|
213
|
+
automatically, and puts a `macos-cu` launcher on your `PATH`.
|
|
179
214
|
|
|
180
215
|
## The four capabilities that matter
|
|
181
216
|
|
|
@@ -275,13 +310,19 @@ Release steps and catalog-listing criteria live in
|
|
|
275
310
|
可视反馈,以及可选的 Jev 语义护栏。
|
|
276
311
|
|
|
277
312
|
```bash
|
|
278
|
-
|
|
279
|
-
macos-
|
|
313
|
+
# 1) 命令行本体(所有集成都调它,也可单独使用)
|
|
314
|
+
pip install macos-computer-use-kit # 或 pipx install macos-computer-use-kit
|
|
315
|
+
macos-cu doctor # 检查辅助功能 / 屏幕录制权限、显示器、依赖、Jev
|
|
316
|
+
|
|
317
|
+
# 2) 选一个 agent 集成
|
|
318
|
+
pi install npm:pi-macos-computer-use # pi
|
|
319
|
+
dsh plugin --profile <名> add dsh-macos-computer-use # DeepSeek Harness
|
|
280
320
|
```
|
|
281
321
|
|
|
282
|
-
|
|
283
|
-
`
|
|
284
|
-
|
|
322
|
+
opencode 的集成走 checkout:`git clone` 后执行 `./install.sh`,它会装 skill、放一个
|
|
323
|
+
`macos-cu` 启动器,并可写入可选的 Jev key。完整流程与避坑见
|
|
324
|
+
[`skill/SKILL.md`](skill/SKILL.md),发布与收录流程见
|
|
325
|
+
[`PUBLISHING.md`](PUBLISHING.md)。
|
|
285
326
|
|
|
286
327
|
## License
|
|
287
328
|
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Publishing this toolkit
|
|
2
2
|
|
|
3
|
+
> Maintainer document, public on purpose: a fork should be able to release under
|
|
4
|
+
> its own npm/PyPI names without reverse-engineering this layout. Nothing here is
|
|
5
|
+
> secret — no credentials, no private infrastructure, no deploy internals.
|
|
6
|
+
|
|
3
7
|
Three artifacts can be published from this repository, all versioned in lockstep
|
|
4
8
|
with the CLI (bump `pyproject.toml`, `packages/pi/package.json`, and
|
|
5
9
|
`packages/dsh/package.json` together):
|
|
@@ -89,12 +93,22 @@ that root-mounts the package. Prefer npm or the tarball for distribution.
|
|
|
89
93
|
## Catalog listings
|
|
90
94
|
|
|
91
95
|
- **pi**: publishing to npm is enough — the gallery at <https://pi.dev/packages>
|
|
92
|
-
lists packages tagged `pi-package`.
|
|
93
|
-
`pi
|
|
96
|
+
lists packages tagged `pi-package`. This is confirmed working: the page for
|
|
97
|
+
`pi-macos-computer-use` appeared shortly after the npm publish. Optionally add
|
|
98
|
+
a `pi.image` or `pi.video` preview to `packages/pi/package.json` for a richer
|
|
99
|
+
card.
|
|
94
100
|
- **dsh**: add the `dsh-plugin` GitHub topic, then open a PR against
|
|
95
|
-
[awesome-dsh-plugin](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin)
|
|
96
|
-
|
|
97
|
-
|
|
101
|
+
[awesome-dsh-plugin](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin).
|
|
102
|
+
Submissions are **one YAML file per plugin** under `data/plugins/` — never edit
|
|
103
|
+
the generated READMEs. For this monorepo the file is
|
|
104
|
+
`data/plugins/Sur-Cai__macos-computer-use-kit--packages-dsh.yml`, with `url`
|
|
105
|
+
pointing at `.../tree/main/packages/dsh` and `name` written as
|
|
106
|
+
`Sur-Cai/macos-computer-use-kit#packages/dsh`.
|
|
107
|
+
|
|
108
|
+
Their CI enforces a **1-day minimum repository age** (ours was created
|
|
109
|
+
2026-09-22T10:22Z, so the earliest valid submission is the next day at the same
|
|
110
|
+
time) and at most 3 entries per PR. Descriptions are checked against the code,
|
|
111
|
+
so keep numbers and tool names exact. There is also
|
|
98
112
|
[dsh-market](https://github.com/dsh-market/dsh-market) for in-app discovery.
|
|
99
113
|
|
|
100
114
|
## Release checklist
|
|
@@ -106,5 +120,18 @@ macos-cu doctor # permissions still granted after reinstalling
|
|
|
106
120
|
(cd packages/dsh && npm run typecheck && npm run build)
|
|
107
121
|
```
|
|
108
122
|
|
|
109
|
-
Then tag (`git tag
|
|
123
|
+
Then tag (`git tag v<version> && git push --tags`) and publish the artifacts you
|
|
110
124
|
intend to ship.
|
|
125
|
+
|
|
126
|
+
## Current release state
|
|
127
|
+
|
|
128
|
+
| Artifact | Published | Notes |
|
|
129
|
+
| --- | --- | --- |
|
|
130
|
+
| `macos-computer-use-kit` | PyPI `0.2.1` | `pip install macos-computer-use-kit` |
|
|
131
|
+
| `pi-macos-computer-use` | npm `0.2.1` | listed on pi.dev/packages |
|
|
132
|
+
| `dsh-macos-computer-use` | npm `0.2.1` | awesome-dsh-plugin PR pending |
|
|
133
|
+
|
|
134
|
+
Known follow-ups: the PyPI `Programming Language :: Python :: 3.14` classifier is
|
|
135
|
+
not set (the wheel installs and runs on 3.14 — verified — but the badge stops at
|
|
136
|
+
3.13 until the next release adds it), and the dsh catalog PR is gated on the
|
|
137
|
+
1-day repository-age rule.
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# macos-computer-use-kit
|
|
2
2
|
|
|
3
|
+
[](https://pypi.org/project/macos-computer-use-kit/)
|
|
4
|
+
[](https://pypi.org/project/macos-computer-use-kit/)
|
|
5
|
+
[](https://www.npmjs.com/package/pi-macos-computer-use)
|
|
6
|
+
[](https://www.npmjs.com/package/dsh-macos-computer-use)
|
|
7
|
+
[](https://github.com/Sur-Cai/macos-computer-use-kit/actions/workflows/ci.yml)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
3
10
|
**AX-first computer use for AI agents on macOS.** Instead of screenshot → eyeball
|
|
4
11
|
coordinates → click and hope, read the accessibility tree, get each element's
|
|
5
12
|
semantics and exact geometry, act on it, then verify the action actually changed
|
|
@@ -10,6 +17,14 @@ window-scoped input, clipboard-safe pasting, action read-back verification,
|
|
|
10
17
|
blank-frame detection, visual feedback, and optional Jev (TypeSafe System One)
|
|
11
18
|
semantic guards.
|
|
12
19
|
|
|
20
|
+
**Jev is the semantic guard layer.** A small model — no reasoning tokens, no
|
|
21
|
+
prose — returns calibrated judgments right before an irreversible action: *is
|
|
22
|
+
this still the intended recipient? does the field hold the intended text? what,
|
|
23
|
+
if anything, blocks the action?* The code, not the model, decides whether to
|
|
24
|
+
proceed, and the model may only ever suggest the two recoveries that cannot send
|
|
25
|
+
or submit anything. Probabilities in, decisions out. See
|
|
26
|
+
[Jev semantic guards](#jev-semantic-guards-optional).
|
|
27
|
+
|
|
13
28
|
Works with any agent that can run a shell command, and ships first-class
|
|
14
29
|
packages for [pi](#pi) and [DeepSeek Harness](#deepseek-harness-dsh).
|
|
15
30
|
|
|
@@ -29,10 +44,19 @@ invented from scratch:
|
|
|
29
44
|
macOS 12+, Python 3.10+.
|
|
30
45
|
|
|
31
46
|
```bash
|
|
32
|
-
|
|
33
|
-
macos-
|
|
47
|
+
# 1) the CLI — every integration below drives this, and it works on its own
|
|
48
|
+
pip install macos-computer-use-kit # or: pipx install macos-computer-use-kit
|
|
49
|
+
macos-cu doctor # permissions, displays, dependencies
|
|
50
|
+
|
|
51
|
+
# 2) your agent integration (optional — pick one)
|
|
52
|
+
pi install npm:pi-macos-computer-use # pi
|
|
53
|
+
dsh plugin --profile <name> add dsh-macos-computer-use # DeepSeek Harness
|
|
54
|
+
```
|
|
34
55
|
|
|
35
|
-
|
|
56
|
+
From a checkout — this is also the opencode integration (editable CLI, the
|
|
57
|
+
`macos-computer-use` skill, a `macos-cu` launcher, and the optional Jev key):
|
|
58
|
+
|
|
59
|
+
```bash
|
|
36
60
|
git clone https://github.com/Sur-Cai/macos-computer-use-kit && cd macos-computer-use-kit
|
|
37
61
|
./install.sh
|
|
38
62
|
```
|
|
@@ -102,23 +126,30 @@ coordinates. That is normal. `macos-cu doctor` prints the layout.
|
|
|
102
126
|
|
|
103
127
|
| Harness | What you get | Install |
|
|
104
128
|
| --- | --- | --- |
|
|
105
|
-
| any agent with a shell | the full CLI | `pip install macos-computer-use-kit` |
|
|
129
|
+
| any agent with a shell | the full CLI ([PyPI](https://pypi.org/project/macos-computer-use-kit/)) | `pip install macos-computer-use-kit` |
|
|
106
130
|
| [opencode](https://opencode.ai) | skill `macos-computer-use` (auto-discovered) | `./install.sh` |
|
|
107
|
-
| [pi](https://pi.dev) | skill + 9 native tools | `pi install npm:pi-macos-computer-use` |
|
|
108
|
-
| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | plugin bundle,
|
|
131
|
+
| [pi](https://pi.dev) | skill + 9 native tools ([npm](https://www.npmjs.com/package/pi-macos-computer-use), [catalog](https://pi.dev/packages/pi-macos-computer-use)) | `pi install npm:pi-macos-computer-use` |
|
|
132
|
+
| [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) | plugin bundle, 6 tools ([npm](https://www.npmjs.com/package/dsh-macos-computer-use)) | `dsh plugin --profile <name> add dsh-macos-computer-use` |
|
|
133
|
+
|
|
134
|
+
Every integration is a thin bridge over the same CLI, so install the CLI first
|
|
135
|
+
(`pip install macos-computer-use-kit`). The three published artifacts — the PyPI
|
|
136
|
+
CLI, the pi package, and the dsh bundle — are versioned and released together;
|
|
137
|
+
the badges at the top of this file show the current release.
|
|
109
138
|
|
|
110
139
|
<a name="pi"></a>
|
|
111
140
|
### pi package
|
|
112
141
|
|
|
113
|
-
`
|
|
142
|
+
[`pi-macos-computer-use`](https://www.npmjs.com/package/pi-macos-computer-use) on
|
|
143
|
+
npm (`pi-package` keyword), sources in `packages/pi`. Registers
|
|
114
144
|
`macos_cu_doctor`, `macos_ax_find`, `macos_ax_press`, `macos_input_windows`,
|
|
115
145
|
`macos_input_click`, `macos_input_key`, `macos_paste`, `macos_shot`,
|
|
116
146
|
`macos_jev_guard`. Every tool shells out with an argv array (`shell: false`), so
|
|
117
147
|
model-supplied text can never reach a shell.
|
|
118
148
|
|
|
119
149
|
```bash
|
|
120
|
-
|
|
121
|
-
pi
|
|
150
|
+
pip install macos-computer-use-kit # the CLI the tools call
|
|
151
|
+
pi install npm:pi-macos-computer-use # the integration
|
|
152
|
+
pi -e ./packages/pi # or try a checkout for one run, without installing
|
|
122
153
|
```
|
|
123
154
|
|
|
124
155
|
App launchers do not inherit your interactive shell's `PATH`. If the CLI is
|
|
@@ -128,13 +159,16 @@ restart pi (the dsh plugin honours the same variable).
|
|
|
128
159
|
<a name="deepseek-harness-dsh"></a>
|
|
129
160
|
### DeepSeek Harness plugin
|
|
130
161
|
|
|
131
|
-
`
|
|
162
|
+
[`dsh-macos-computer-use`](https://www.npmjs.com/package/dsh-macos-computer-use)
|
|
163
|
+
on npm, sources in `packages/dsh` — a Cordis bundle
|
|
132
164
|
(`dsh.bundle.patch` → `cordis.patch.yml`). Registers `macos_cu_doctor`,
|
|
133
|
-
`macos_ax_find`, `macos_ax_press`, `macos_input_click`, `macos_shot
|
|
165
|
+
`macos_ax_find`, `macos_ax_press`, `macos_input_click`, `macos_shot`,
|
|
166
|
+
`macos_jev_guard` (the same optional Jev guard the pi package exposes).
|
|
134
167
|
|
|
135
168
|
```bash
|
|
136
|
-
|
|
137
|
-
dsh --profile demo
|
|
169
|
+
pip install macos-computer-use-kit # the CLI the tools call
|
|
170
|
+
dsh plugin --profile demo add dsh-macos-computer-use # the plugin
|
|
171
|
+
dsh --profile demo --dump-config # verify the layer before booting
|
|
138
172
|
```
|
|
139
173
|
|
|
140
174
|
It deliberately does **not** claim the exclusive `ctx.computerUse` provider slot:
|
|
@@ -145,7 +179,7 @@ in-box Cua Driver provider. See `packages/dsh/README.md`.
|
|
|
145
179
|
|
|
146
180
|
`./install.sh` installs `skill/SKILL.md` to
|
|
147
181
|
`~/.config/opencode/skills/macos-computer-use/`, where opencode discovers it
|
|
148
|
-
automatically
|
|
182
|
+
automatically, and puts a `macos-cu` launcher on your `PATH`.
|
|
149
183
|
|
|
150
184
|
## The four capabilities that matter
|
|
151
185
|
|
|
@@ -245,13 +279,19 @@ Release steps and catalog-listing criteria live in
|
|
|
245
279
|
可视反馈,以及可选的 Jev 语义护栏。
|
|
246
280
|
|
|
247
281
|
```bash
|
|
248
|
-
|
|
249
|
-
macos-
|
|
282
|
+
# 1) 命令行本体(所有集成都调它,也可单独使用)
|
|
283
|
+
pip install macos-computer-use-kit # 或 pipx install macos-computer-use-kit
|
|
284
|
+
macos-cu doctor # 检查辅助功能 / 屏幕录制权限、显示器、依赖、Jev
|
|
285
|
+
|
|
286
|
+
# 2) 选一个 agent 集成
|
|
287
|
+
pi install npm:pi-macos-computer-use # pi
|
|
288
|
+
dsh plugin --profile <名> add dsh-macos-computer-use # DeepSeek Harness
|
|
250
289
|
```
|
|
251
290
|
|
|
252
|
-
|
|
253
|
-
`
|
|
254
|
-
|
|
291
|
+
opencode 的集成走 checkout:`git clone` 后执行 `./install.sh`,它会装 skill、放一个
|
|
292
|
+
`macos-cu` 启动器,并可写入可选的 Jev key。完整流程与避坑见
|
|
293
|
+
[`skill/SKILL.md`](skill/SKILL.md),发布与收录流程见
|
|
294
|
+
[`PUBLISHING.md`](PUBLISHING.md)。
|
|
255
295
|
|
|
256
296
|
## License
|
|
257
297
|
|
|
@@ -4,7 +4,7 @@ DeepSeek Harness (dsh) bundle bridging dsh agents to the
|
|
|
4
4
|
[macos-computer-use-kit](https://github.com/Sur-Cai/macos-computer-use-kit)
|
|
5
5
|
`macos-cu` CLI: AX-first computer use on macOS.
|
|
6
6
|
|
|
7
|
-
The plugin registers
|
|
7
|
+
The plugin registers six focused tools that shell out to `macos-cu` with an
|
|
8
8
|
argument array (never shell-string concatenation) and return the CLI's JSON
|
|
9
9
|
output:
|
|
10
10
|
|
|
@@ -15,6 +15,7 @@ output:
|
|
|
15
15
|
| `macos_ax_press` | `macos-cu ax press\|setvalue` | Native AX action with read-back verification |
|
|
16
16
|
| `macos_input_click` | `macos-cu input click` | Window-scoped click that never moves the user's cursor |
|
|
17
17
|
| `macos_shot` | `macos-cu shot capture\|check\|windows` | Blank-frame-checked screenshots for verification only |
|
|
18
|
+
| `macos_jev_guard` | `macos-cu jev guard\|select` | Optional Jev semantic guard before an irreversible action (needs `TYPESAFE_API_KEY`) |
|
|
18
19
|
|
|
19
20
|
The AX-first intent is baked into every tool description: locate elements via
|
|
20
21
|
`macos_ax_find` and use the returned geometry — use instead of guessing
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-macos-computer-use",
|
|
3
|
-
"version": "0.2.
|
|
4
|
-
"description": "AX-first computer use on macOS for DeepSeek Harness: accessibility-tree targeting instead of screenshot coordinate guessing, window-scoped input, action read-back verification,
|
|
3
|
+
"version": "0.2.2",
|
|
4
|
+
"description": "AX-first computer use on macOS for DeepSeek Harness: accessibility-tree targeting instead of screenshot coordinate guessing, window-scoped input, action read-back verification, blank-frame-checked screenshots, and optional Jev (TypeSafe System One) semantic guards before irreversible actions \u2014 bridged to the macos-cu CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"types": "lib/index.d.ts",
|
|
@@ -31,7 +31,10 @@
|
|
|
31
31
|
"accessibility-tree",
|
|
32
32
|
"automation",
|
|
33
33
|
"gui-automation",
|
|
34
|
-
"agent-tools"
|
|
34
|
+
"agent-tools",
|
|
35
|
+
"jev",
|
|
36
|
+
"typesafe-ai",
|
|
37
|
+
"system-one-models"
|
|
35
38
|
],
|
|
36
39
|
"homepage": "https://github.com/Sur-Cai/macos-computer-use-kit#readme",
|
|
37
40
|
"repository": {
|
|
@@ -148,6 +148,44 @@ const shot = defineTool({
|
|
|
148
148
|
},
|
|
149
149
|
})
|
|
150
150
|
|
|
151
|
+
const jevGuard = defineTool({
|
|
152
|
+
name: 'macos_jev_guard',
|
|
153
|
+
description: `Optional Jev (TypeSafe System One) semantic guard for the moment before an irreversible action. \`macos-cu jev guard\` fans one request out into calibrated judgments about expected versus observed state (right_target, input_ok, blocker, next_action) plus a decision, and \`macos-cu jev select\` picks one candidate element id with a confidence gate and a 'none' escape hatch. Use it only for semantic identity/state/effect questions that ordinary code cannot decide — never for blank-frame detection or signature comparison. Requires TYPESAFE_API_KEY or ~/.config/typesafe/api_key; the rest of this plugin works without it. The JSON request is sent on stdin.`,
|
|
154
|
+
parameters: {
|
|
155
|
+
command: {
|
|
156
|
+
type: 'string',
|
|
157
|
+
required: true,
|
|
158
|
+
enum: ['guard', 'select'],
|
|
159
|
+
description: 'guard = pre-action judgment; select = pick an element id from candidates.',
|
|
160
|
+
},
|
|
161
|
+
payload: {
|
|
162
|
+
type: 'string',
|
|
163
|
+
required: true,
|
|
164
|
+
description:
|
|
165
|
+
'JSON request for the CLI stdin. guard: {"task":...,"expected":{...},"observed":{...}}. select: {"goal":...,"candidates":[{"id":...,"text":...}]}.',
|
|
166
|
+
},
|
|
167
|
+
},
|
|
168
|
+
output: OUTPUT,
|
|
169
|
+
async execute(args, exec) {
|
|
170
|
+
let parsed: unknown
|
|
171
|
+
try {
|
|
172
|
+
parsed = JSON.parse(args.payload)
|
|
173
|
+
} catch (error) {
|
|
174
|
+
throw new Error(`macos_jev_guard 'payload' is not valid JSON: ${(error as Error).message}`)
|
|
175
|
+
}
|
|
176
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
177
|
+
throw new Error("macos_jev_guard 'payload' must be a JSON object.")
|
|
178
|
+
}
|
|
179
|
+
if (args.command === 'guard' && !(parsed as { task?: unknown }).task) {
|
|
180
|
+
throw new Error("macos_jev_guard command='guard' needs a 'task' field in the payload.")
|
|
181
|
+
}
|
|
182
|
+
if (args.command === 'select' && !Array.isArray((parsed as { candidates?: unknown }).candidates)) {
|
|
183
|
+
throw new Error("macos_jev_guard command='select' needs a 'candidates' array in the payload.")
|
|
184
|
+
}
|
|
185
|
+
return runMacosCu(['jev', args.command], exec.signal, JSON.stringify(parsed))
|
|
186
|
+
},
|
|
187
|
+
})
|
|
188
|
+
|
|
151
189
|
/** Register the macos-cu bridge tools. Tool registrations auto-dispose on unload. */
|
|
152
190
|
export function apply(ctx: Context): void {
|
|
153
191
|
ctx.tools.register(doctor)
|
|
@@ -155,4 +193,5 @@ export function apply(ctx: Context): void {
|
|
|
155
193
|
ctx.tools.register(axPress)
|
|
156
194
|
ctx.tools.register(inputClick)
|
|
157
195
|
ctx.tools.register(shot)
|
|
196
|
+
ctx.tools.register(jevGuard)
|
|
158
197
|
}
|
|
@@ -35,13 +35,16 @@ export function binary(): string {
|
|
|
35
35
|
* Run the CLI with an argument array (never a shell string) and return the
|
|
36
36
|
* outcome. Missing-binary (ENOENT) resolves to an actionable install message
|
|
37
37
|
* instead of throwing, so the model sees what to do next.
|
|
38
|
+
*
|
|
39
|
+
* `stdin` is written and the stream closed immediately: the `jev` subcommand
|
|
40
|
+
* reads a JSON payload from stdin and would otherwise wait forever.
|
|
38
41
|
*/
|
|
39
|
-
export async function runMacosCu(argv: string[], signal: AbortSignal): Promise<string> {
|
|
42
|
+
export async function runMacosCu(argv: string[], signal: AbortSignal, stdin?: string): Promise<string> {
|
|
40
43
|
if (signal.aborted) throw new Error('macos-cu call aborted before spawn')
|
|
41
44
|
const bin = binary()
|
|
42
45
|
const outcome = await new Promise<{ code: number; stdout: string; stderr: string; spawnError?: NodeJS.ErrnoException }>(
|
|
43
46
|
(resolve) => {
|
|
44
|
-
execFile(
|
|
47
|
+
const child = execFile(
|
|
45
48
|
bin,
|
|
46
49
|
argv,
|
|
47
50
|
{ timeout: 55_000, signal, maxBuffer: 8 * 1024 * 1024, windowsHide: true },
|
|
@@ -57,6 +60,7 @@ export async function runMacosCu(argv: string[], signal: AbortSignal): Promise<s
|
|
|
57
60
|
resolve({ code, stdout: String(stdout), stderr: String(stderr) })
|
|
58
61
|
},
|
|
59
62
|
)
|
|
63
|
+
child.stdin?.end(stdin ?? '')
|
|
60
64
|
},
|
|
61
65
|
)
|
|
62
66
|
if (outcome.spawnError !== undefined) return MISSING_BINARY_MESSAGE
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-macos-computer-use",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "AX-first computer use on macOS for pi: accessibility-tree targeting instead of screenshot coordinate guessing, window-scoped input that never moves the user's cursor, clipboard-safe pasting, action read-back verification, blank-frame detection, and optional Jev semantic guards.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -14,7 +14,10 @@
|
|
|
14
14
|
"accessibility-tree",
|
|
15
15
|
"automation",
|
|
16
16
|
"gui-automation",
|
|
17
|
-
"agent-tools"
|
|
17
|
+
"agent-tools",
|
|
18
|
+
"jev",
|
|
19
|
+
"typesafe-ai",
|
|
20
|
+
"system-one-models"
|
|
18
21
|
],
|
|
19
22
|
"homepage": "https://github.com/Sur-Cai/macos-computer-use-kit#readme",
|
|
20
23
|
"repository": {
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "macos-computer-use-kit"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "AX-first computer-use toolkit for AI agents on macOS: accessibility-tree targeting, process-scoped input, clipboard-safe paste, action read-back verification, visual feedback, and optional Jev semantic guards."
|
|
9
9
|
license = "MIT"
|
|
10
10
|
license-files = ["LICENSE"]
|
|
@@ -20,6 +20,10 @@ keywords = [
|
|
|
20
20
|
"opencode",
|
|
21
21
|
"pi-package",
|
|
22
22
|
"dsh-plugin",
|
|
23
|
+
"jev",
|
|
24
|
+
"typesafe-ai",
|
|
25
|
+
"system-one",
|
|
26
|
+
"llm-guardrails",
|
|
23
27
|
]
|
|
24
28
|
classifiers = [
|
|
25
29
|
"Development Status :: 4 - Beta",
|
|
@@ -32,6 +36,7 @@ classifiers = [
|
|
|
32
36
|
"Programming Language :: Python :: 3.11",
|
|
33
37
|
"Programming Language :: Python :: 3.12",
|
|
34
38
|
"Programming Language :: Python :: 3.13",
|
|
39
|
+
"Programming Language :: Python :: 3.14",
|
|
35
40
|
"Topic :: Software Development :: User Interfaces",
|
|
36
41
|
]
|
|
37
42
|
dependencies = [
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/packages/dsh/package-lock.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/skill/reference/jev-best-practices.md
RENAMED
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/__main__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/darwin.py
RENAMED
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/input_events.py
RENAMED
|
File without changes
|
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/overlay.py
RENAMED
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/paste.py
RENAMED
|
File without changes
|
{macos_computer_use_kit-0.2.1 → macos_computer_use_kit-0.2.2}/src/macos_computer_use/shot.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|