ventriloquist-mcp 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ventriloquist_mcp-0.2.0/.github/ISSUE_TEMPLATE/pack-report.yml +48 -0
- ventriloquist_mcp-0.2.0/.github/workflows/release.yml +43 -0
- ventriloquist_mcp-0.2.0/.github/workflows/test.yml +57 -0
- ventriloquist_mcp-0.2.0/.gitignore +6 -0
- ventriloquist_mcp-0.2.0/CONTRIBUTING.md +101 -0
- ventriloquist_mcp-0.2.0/LICENSE +21 -0
- ventriloquist_mcp-0.2.0/PKG-INFO +379 -0
- ventriloquist_mcp-0.2.0/README.md +347 -0
- ventriloquist_mcp-0.2.0/docs/ARCHITECTURE.md +367 -0
- ventriloquist_mcp-0.2.0/docs/DEMO.md +93 -0
- ventriloquist_mcp-0.2.0/docs/HANDOFF.md +231 -0
- ventriloquist_mcp-0.2.0/docs/SECURITY.md +274 -0
- ventriloquist_mcp-0.2.0/docs/architecture.svg +104 -0
- ventriloquist_mcp-0.2.0/llms.txt +280 -0
- ventriloquist_mcp-0.2.0/pyproject.toml +56 -0
- ventriloquist_mcp-0.2.0/server.json +42 -0
- ventriloquist_mcp-0.2.0/tests/__init__.py +0 -0
- ventriloquist_mcp-0.2.0/tests/fakes.py +158 -0
- ventriloquist_mcp-0.2.0/tests/install_smoke.py +52 -0
- ventriloquist_mcp-0.2.0/tests/mcp_smoke.py +50 -0
- ventriloquist_mcp-0.2.0/tests/test_anchors.py +245 -0
- ventriloquist_mcp-0.2.0/tests/test_cli.py +39 -0
- ventriloquist_mcp-0.2.0/tests/test_explorer.py +277 -0
- ventriloquist_mcp-0.2.0/tests/test_harness.py +148 -0
- ventriloquist_mcp-0.2.0/tests/test_heal.py +247 -0
- ventriloquist_mcp-0.2.0/tests/test_llm_cli.py +103 -0
- ventriloquist_mcp-0.2.0/tests/test_llm_sdk.py +72 -0
- ventriloquist_mcp-0.2.0/tests/test_packs.py +146 -0
- ventriloquist_mcp-0.2.0/tests/test_paths.py +66 -0
- ventriloquist_mcp-0.2.0/tests/test_policy.py +111 -0
- ventriloquist_mcp-0.2.0/tests/test_release.py +30 -0
- ventriloquist_mcp-0.2.0/tests/test_runtime.py +270 -0
- ventriloquist_mcp-0.2.0/tests/test_server.py +142 -0
- ventriloquist_mcp-0.2.0/vent/__init__.py +3 -0
- ventriloquist_mcp-0.2.0/vent/anchors.py +387 -0
- ventriloquist_mcp-0.2.0/vent/ax.py +358 -0
- ventriloquist_mcp-0.2.0/vent/bundled_packs/com.apple.Notes/pack.json +315 -0
- ventriloquist_mcp-0.2.0/vent/bundled_packs/com.apple.TextEdit/pack.json +186 -0
- ventriloquist_mcp-0.2.0/vent/bundled_packs/com.microsoft.VSCode/pack.json +2514 -0
- ventriloquist_mcp-0.2.0/vent/bundled_packs/com.tinyspeck.slackmacgap/pack.json +1072 -0
- ventriloquist_mcp-0.2.0/vent/cli.py +577 -0
- ventriloquist_mcp-0.2.0/vent/compiler.py +271 -0
- ventriloquist_mcp-0.2.0/vent/explorer.py +313 -0
- ventriloquist_mcp-0.2.0/vent/harness.py +445 -0
- ventriloquist_mcp-0.2.0/vent/heal.py +314 -0
- ventriloquist_mcp-0.2.0/vent/llm.py +208 -0
- ventriloquist_mcp-0.2.0/vent/packs.py +309 -0
- ventriloquist_mcp-0.2.0/vent/paths.py +39 -0
- ventriloquist_mcp-0.2.0/vent/policy.py +217 -0
- ventriloquist_mcp-0.2.0/vent/runtime.py +417 -0
- ventriloquist_mcp-0.2.0/vent/server.py +222 -0
- ventriloquist_mcp-0.2.0/vent/snapshot.py +282 -0
- ventriloquist_mcp-0.2.0/vent/untrusted.py +37 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: Pack report
|
|
2
|
+
description: Tell us how a pack behaved on your Mac, working or not.
|
|
3
|
+
title: "Pack report: <app> on macOS <version>"
|
|
4
|
+
labels: ["pack report"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
Packs have only been measured on the author's Mac, so reports from
|
|
10
|
+
other machines are the most useful thing you can send, including
|
|
11
|
+
"it just worked".
|
|
12
|
+
- type: input
|
|
13
|
+
id: app
|
|
14
|
+
attributes:
|
|
15
|
+
label: App and version
|
|
16
|
+
placeholder: Visual Studio Code 1.133.0
|
|
17
|
+
validations:
|
|
18
|
+
required: true
|
|
19
|
+
- type: input
|
|
20
|
+
id: macos
|
|
21
|
+
attributes:
|
|
22
|
+
label: macOS version
|
|
23
|
+
placeholder: "26.3"
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
- type: input
|
|
27
|
+
id: vent
|
|
28
|
+
attributes:
|
|
29
|
+
label: vent version
|
|
30
|
+
description: Output of `vent --version`.
|
|
31
|
+
validations:
|
|
32
|
+
required: true
|
|
33
|
+
- type: textarea
|
|
34
|
+
id: report
|
|
35
|
+
attributes:
|
|
36
|
+
label: vent verify report
|
|
37
|
+
description: |
|
|
38
|
+
Run `vent verify <app> --report out.json` and paste out.json here.
|
|
39
|
+
It holds no pack contents and no model output; the only app content
|
|
40
|
+
is each resolved element's live label. Read it before pasting.
|
|
41
|
+
render: json
|
|
42
|
+
- type: textarea
|
|
43
|
+
id: what-happened
|
|
44
|
+
attributes:
|
|
45
|
+
label: What happened
|
|
46
|
+
description: Which tools you ran, through which client, and what you saw.
|
|
47
|
+
validations:
|
|
48
|
+
required: true
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v7
|
|
16
|
+
- uses: actions/setup-python@v7
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.13"
|
|
19
|
+
- name: Check the tag matches the package version
|
|
20
|
+
run: |
|
|
21
|
+
version=$(python -c "import pathlib, re; print(re.search(r'__version__ = \"(.+?)\"', pathlib.Path('vent/__init__.py').read_text()).group(1))")
|
|
22
|
+
test "v$version" = "$GITHUB_REF_NAME" || { echo "Tag $GITHUB_REF_NAME does not match version $version"; exit 1; }
|
|
23
|
+
- run: python -m pip install --upgrade build
|
|
24
|
+
- run: python -m build
|
|
25
|
+
- uses: actions/upload-artifact@v7
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
publish:
|
|
31
|
+
needs: build
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
environment:
|
|
34
|
+
name: pypi
|
|
35
|
+
url: https://pypi.org/p/ventriloquist-mcp
|
|
36
|
+
permissions:
|
|
37
|
+
id-token: write
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/download-artifact@v8
|
|
40
|
+
with:
|
|
41
|
+
name: dist
|
|
42
|
+
path: dist/
|
|
43
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
pull_request:
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
# The offline suite is designed to run without macOS, permissions, or model
|
|
8
|
+
# credentials: fake accessibility trees and a scripted-model seam. Running
|
|
9
|
+
# it on Linux is the proof of that claim, not just a convenience.
|
|
10
|
+
offline:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.14"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v7
|
|
17
|
+
- uses: actions/setup-python@v7
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python-version }}
|
|
20
|
+
- run: python -m venv .venv && .venv/bin/pip install -e . pytest
|
|
21
|
+
- run: .venv/bin/python -m pytest tests/ -q
|
|
22
|
+
|
|
23
|
+
# A second run on macOS exercises the same suite where the real pyobjc
|
|
24
|
+
# accessibility bindings are importable, catching anything that only breaks
|
|
25
|
+
# when the AX layer is genuinely present.
|
|
26
|
+
macos:
|
|
27
|
+
runs-on: macos-latest
|
|
28
|
+
strategy:
|
|
29
|
+
matrix:
|
|
30
|
+
python-version: ["3.11", "3.14"]
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v7
|
|
33
|
+
- uses: actions/setup-python@v7
|
|
34
|
+
with:
|
|
35
|
+
python-version: ${{ matrix.python-version }}
|
|
36
|
+
- run: python -m venv .venv && .venv/bin/pip install -e . pytest
|
|
37
|
+
- run: .venv/bin/python -m pytest tests/ -q
|
|
38
|
+
|
|
39
|
+
# Builds the wheel, installs it into a clean environment the way users get
|
|
40
|
+
# vent, and lists the served tools over MCP stdio. An editable install reads
|
|
41
|
+
# files straight from the checkout, so only this job can catch a wheel that
|
|
42
|
+
# leaves the bundled packs or an entry point behind.
|
|
43
|
+
install:
|
|
44
|
+
strategy:
|
|
45
|
+
matrix:
|
|
46
|
+
os: [ubuntu-latest, macos-latest]
|
|
47
|
+
runs-on: ${{ matrix.os }}
|
|
48
|
+
steps:
|
|
49
|
+
- uses: actions/checkout@v7
|
|
50
|
+
- uses: actions/setup-python@v7
|
|
51
|
+
with:
|
|
52
|
+
python-version: "3.13"
|
|
53
|
+
- run: python -m pip wheel --no-deps --wheel-dir dist .
|
|
54
|
+
- run: python -m venv "$RUNNER_TEMP/env" && "$RUNNER_TEMP/env/bin/pip" install dist/*.whl
|
|
55
|
+
- name: Serve the bundled packs from the installed wheel
|
|
56
|
+
working-directory: ${{ runner.temp }}
|
|
57
|
+
run: '"$RUNNER_TEMP/env/bin/python" "$GITHUB_WORKSPACE/tests/install_smoke.py" "$RUNNER_TEMP/env/bin"'
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Contributing to Ventriloquist
|
|
2
|
+
|
|
3
|
+
Thanks for looking at this. A few things will make a change land smoothly,
|
|
4
|
+
and some of them are load-bearing rather than stylistic.
|
|
5
|
+
|
|
6
|
+
## Getting set up
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
python3 -m venv .venv
|
|
10
|
+
.venv/bin/pip install -e . pytest
|
|
11
|
+
.venv/bin/python -m pytest tests/
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The test suite runs with no Mac permissions, no live apps, and no model
|
|
15
|
+
credentials. It uses in-memory fake accessibility trees (`tests/fakes.py`)
|
|
16
|
+
and a scripted-model seam (`llm.set_completer_for_tests`), so it runs the
|
|
17
|
+
same on your laptop and on Linux CI. If a change you make cannot be tested
|
|
18
|
+
that way, that is usually a sign the logic should move behind the `Element`
|
|
19
|
+
interface or the `llm` seam rather than a sign the test is impossible.
|
|
20
|
+
|
|
21
|
+
Live behavior (the `ax` layer, `vent inspect`, `vent run` against a real
|
|
22
|
+
app) needs macOS and Accessibility permission for your terminal, granted in
|
|
23
|
+
System Settings, Privacy & Security, Accessibility.
|
|
24
|
+
|
|
25
|
+
An editable install reads files straight from the checkout, so it cannot
|
|
26
|
+
show a packaging mistake. CI also builds the wheel, installs it into a
|
|
27
|
+
clean environment, and runs `tests/install_smoke.py` against it. To do
|
|
28
|
+
the same locally:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
python3 -m venv --clear /tmp/vent-env
|
|
32
|
+
/tmp/vent-env/bin/pip install . # a regular install, built as a wheel
|
|
33
|
+
cd /tmp && /tmp/vent-env/bin/python "$OLDPWD/tests/install_smoke.py" /tmp/vent-env/bin
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Read these first
|
|
37
|
+
|
|
38
|
+
- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) explains every module and why
|
|
39
|
+
it exists. Read it before the code.
|
|
40
|
+
- [docs/SECURITY.md](docs/SECURITY.md) is the threat model. It is not
|
|
41
|
+
aspirational; the code implements it, and several of its rules are checked
|
|
42
|
+
in tests.
|
|
43
|
+
|
|
44
|
+
## The rules that are not negotiable
|
|
45
|
+
|
|
46
|
+
These are the load-bearing invariants. A change that breaks one is wrong
|
|
47
|
+
even if the tests pass, so the tests exist to make breaking them loud.
|
|
48
|
+
|
|
49
|
+
1. **The action space stays closed.** The runtime executes a fixed set of
|
|
50
|
+
typed ops. There is no `exec`, no shell, no AppleScript built from model
|
|
51
|
+
output. Adding an op means updating SECURITY.md T1 first, then the
|
|
52
|
+
validator in `packs.py`, then the runtime.
|
|
53
|
+
2. **`runtime.py` and `server.py` never import `llm.py`.** The deterministic
|
|
54
|
+
half does not depend on a model. Healing reaches it only through the
|
|
55
|
+
callback `runtime.execute` is handed.
|
|
56
|
+
3. **The model composes, it never authors.** The compiler builds tools only
|
|
57
|
+
from actions the explorer actually executed. Model text is shown to the
|
|
58
|
+
human but never becomes a step.
|
|
59
|
+
4. **Policy screens after the model, never before.** A model cannot approve
|
|
60
|
+
its own action past the policy layer.
|
|
61
|
+
5. **Healed anchors are quarantined, not silently promoted.** Only
|
|
62
|
+
`vent verify`, under a human, rewrites a tool's anchor of record.
|
|
63
|
+
6. **No screenshots, no pixel coordinates, no vision.** If a feature seems
|
|
64
|
+
to need them, redesign the feature.
|
|
65
|
+
|
|
66
|
+
## Shipping a pack
|
|
67
|
+
|
|
68
|
+
Packs you record land in `~/.ventriloquist/packs/<bundle id>/pack.json`
|
|
69
|
+
(or under `VENT_HOME`). To ship one with vent:
|
|
70
|
+
|
|
71
|
+
1. Run `vent verify <app>` and the tool-level harness
|
|
72
|
+
(`vent harness <app> --tools`) and put the numbers in the PR.
|
|
73
|
+
2. Scrub anything personal from the anchors: window titles, workspace
|
|
74
|
+
and account names, document names. Labels in ancestor chains leak
|
|
75
|
+
these too.
|
|
76
|
+
3. Copy the file to `vent/bundled_packs/<bundle id>/pack.json`.
|
|
77
|
+
`test_shipped_packs_all_load` checks that it validates.
|
|
78
|
+
4. Add a row to the "What ships" table in the README.
|
|
79
|
+
|
|
80
|
+
## Style
|
|
81
|
+
|
|
82
|
+
- Plain prose in docs and comments. No em dashes; a comma or a period does
|
|
83
|
+
the same work.
|
|
84
|
+
- Match the surrounding code: its naming, its comment density, its idiom.
|
|
85
|
+
- A comment should say something the code cannot. Skip comments that
|
|
86
|
+
restate the next line.
|
|
87
|
+
- Tests assert behavior, not implementation details, and each test name
|
|
88
|
+
says what property it pins.
|
|
89
|
+
|
|
90
|
+
## Before you open a PR
|
|
91
|
+
|
|
92
|
+
- `.venv/bin/python -m pytest tests/` passes.
|
|
93
|
+
- New ops, new model prompts, or new network calls come with a SECURITY.md
|
|
94
|
+
update in the same PR.
|
|
95
|
+
- If you touched the healer, the explorer, the policy, or the anchor
|
|
96
|
+
resolver, add a test that would fail without your change.
|
|
97
|
+
|
|
98
|
+
Ventriloquist has been built one phase at a time, each phase adversarially
|
|
99
|
+
reviewed before the next began. A PR that keeps that bar (a clear
|
|
100
|
+
invariant, a test that pins it, an honest note on what it does not cover)
|
|
101
|
+
is easy to accept.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shaan Gurushankar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ventriloquist-mcp
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Turn Mac apps into MCP servers using the accessibility API. Explore an app once, then every tool call is deterministic replay: no screenshots, no model per step.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ShaanGS/ventriloquist
|
|
6
|
+
Project-URL: Documentation, https://github.com/ShaanGS/ventriloquist/blob/main/docs/ARCHITECTURE.md
|
|
7
|
+
Project-URL: Issues, https://github.com/ShaanGS/ventriloquist/issues
|
|
8
|
+
Author: Shaan Gurushankar
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: accessibility,agents,automation,computer-use,macos,mcp,model-context-protocol
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Environment :: MacOS X
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Utilities
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: anyio>=4.0
|
|
27
|
+
Requires-Dist: click>=8.1
|
|
28
|
+
Requires-Dist: mcp<3,>=2.0
|
|
29
|
+
Requires-Dist: pyobjc-framework-applicationservices>=10.0; sys_platform == 'darwin'
|
|
30
|
+
Requires-Dist: pyobjc-framework-cocoa>=10.0; sys_platform == 'darwin'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
<div align="center">
|
|
34
|
+
|
|
35
|
+
# Ventriloquist
|
|
36
|
+
|
|
37
|
+
Turn a Mac app into an MCP server using the accessibility API.
|
|
38
|
+
|
|
39
|
+
[Try it in a minute](#try-it-in-a-minute) · [Use from Claude, Cursor or Codex](#use-it-from-claude-cursor-or-codex) · [How it works](#how-it-works)
|
|
40
|
+
|
|
41
|
+
</div>
|
|
42
|
+
|
|
43
|
+
An agent explores the app once, the interesting actions get compiled into a JSON
|
|
44
|
+
"pack" of typed tools, and after that each tool call is deterministic
|
|
45
|
+
replay. No model runs per UI step and none runs on the serving path. The
|
|
46
|
+
client composing tool calls is still a model, so a multi-step workflow
|
|
47
|
+
has one model in it, but it's deciding which tools to call, not where to
|
|
48
|
+
click. No screenshots and no pixel coordinates anywhere in the codebase.
|
|
49
|
+
|
|
50
|
+
<!-- Demo GIF goes here once recorded. Plan and commands: docs/DEMO.md -->
|
|
51
|
+
<!-- mcp-name: io.github.ShaanGS/ventriloquist -->
|
|
52
|
+
|
|
53
|
+
## Try it in a minute
|
|
54
|
+
|
|
55
|
+
Needs macOS and [uv](https://docs.astral.sh/uv/) (`brew install uv`).
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
uv tool install git+https://github.com/ShaanGS/ventriloquist
|
|
59
|
+
vent doctor
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`vent doctor` checks the one permission vent needs. If it says
|
|
63
|
+
Accessibility is missing, open System Settings > Privacy & Security >
|
|
64
|
+
Accessibility, switch on your terminal app, restart the terminal, and run
|
|
65
|
+
it again. (`pipx install git+https://github.com/ShaanGS/ventriloquist`
|
|
66
|
+
works too.)
|
|
67
|
+
|
|
68
|
+
Now drive TextEdit with the pack that ships with vent:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
echo "hello" > /tmp/vent-demo.txt && open -a TextEdit /tmp/vent-demo.txt
|
|
72
|
+
vent run TextEdit write_document --arg text="typed by a pack"
|
|
73
|
+
vent run TextEdit read_document
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
The document changes with no keystrokes and no screenshot: the pack sets
|
|
77
|
+
the text area's value through the accessibility API, then reads it back
|
|
78
|
+
to verify. `write_document` replaces the whole text of the frontmost
|
|
79
|
+
TextEdit document, which is why the first line opens a scratch file; don't
|
|
80
|
+
run it with a document you care about in front.
|
|
81
|
+
|
|
82
|
+
## Use it from Claude, Cursor, or Codex
|
|
83
|
+
|
|
84
|
+
`vent serve` speaks MCP over stdio. GUI apps start MCP servers without
|
|
85
|
+
your shell's PATH, so the config needs vent's absolute path, and
|
|
86
|
+
`vent mcp-config` prints the entry with the right path for your machine:
|
|
87
|
+
|
|
88
|
+
```json
|
|
89
|
+
{
|
|
90
|
+
"mcpServers": {
|
|
91
|
+
"ventriloquist": {
|
|
92
|
+
"command": "/Users/you/.local/bin/vent",
|
|
93
|
+
"args": ["serve"]
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
- **Claude Desktop**: merge it into
|
|
100
|
+
`~/Library/Application Support/Claude/claude_desktop_config.json`.
|
|
101
|
+
- **Cursor**: merge it into `~/.cursor/mcp.json`.
|
|
102
|
+
- **Claude Code**: `claude mcp add --scope user ventriloquist -- ~/.local/bin/vent serve`
|
|
103
|
+
- **Codex CLI**: add to `~/.codex/config.toml`:
|
|
104
|
+
|
|
105
|
+
```toml
|
|
106
|
+
[mcp_servers.ventriloquist]
|
|
107
|
+
command = "/Users/you/.local/bin/vent"
|
|
108
|
+
args = ["serve"]
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The Accessibility permission belongs to whichever app starts
|
|
112
|
+
`vent serve`: Claude Desktop or Cursor themselves, or your terminal for
|
|
113
|
+
Claude Code and Codex. Switch it on for that app (use the + button if it
|
|
114
|
+
isn't in the list yet) and restart it. If every call fails with "not
|
|
115
|
+
trusted for Accessibility", that's the fix.
|
|
116
|
+
|
|
117
|
+
Tools show up as `textedit_write_document`, `code_show_search`,
|
|
118
|
+
`slack_filter_channels`, and so on. Then ask for things in plain words,
|
|
119
|
+
like "open the Search view in VS Code and search the workspace for
|
|
120
|
+
TODO". Anything marked `risk: high` in a pack is refused unless the
|
|
121
|
+
server was started with `VENT_ALLOW_HIGH=1`;
|
|
122
|
+
[docs/SECURITY.md](https://github.com/ShaanGS/ventriloquist/blob/main/docs/SECURITY.md)
|
|
123
|
+
explains why.
|
|
124
|
+
|
|
125
|
+
## What ships
|
|
126
|
+
|
|
127
|
+
| App | Tools | Notes |
|
|
128
|
+
| --- | --- | --- |
|
|
129
|
+
| TextEdit | `write_document`, `read_document` | Acts on the frontmost document; `write_document` replaces all of its text |
|
|
130
|
+
| Notes | `create_note`, `read_open_note` | |
|
|
131
|
+
| VS Code | `show_explorer`, `show_search`, `show_source_control`, `search_workspace` | Launch VS Code with `--force-renderer-accessibility` |
|
|
132
|
+
| Slack | `show_home`, `filter_channels` | Same flag as VS Code |
|
|
133
|
+
|
|
134
|
+
Electron apps like VS Code and Slack only publish a working tree when
|
|
135
|
+
started with that flag, for example
|
|
136
|
+
`open -a "Visual Studio Code" --args --force-renderer-accessibility`.
|
|
137
|
+
Without it the tree can still be read, but actions get silently ignored.
|
|
138
|
+
|
|
139
|
+
Anchors are recorded per machine and per app version, so the bundled
|
|
140
|
+
packs are working examples of the format more than guarantees for your
|
|
141
|
+
exact setup. If a tool refuses to resolve, that's the safety design doing
|
|
142
|
+
its job (it never guesses between lookalike elements), and re-recording
|
|
143
|
+
takes about a minute with `vent inspect` and `vent anchor`. See section 5
|
|
144
|
+
of [docs/ARCHITECTURE.md](https://github.com/ShaanGS/ventriloquist/blob/main/docs/ARCHITECTURE.md).
|
|
145
|
+
|
|
146
|
+
## How it works
|
|
147
|
+
|
|
148
|
+

|
|
149
|
+
|
|
150
|
+
Most computer-use agents are interpreters. A model looks at your screen
|
|
151
|
+
and decides every single click, every time, which is slow and expensive
|
|
152
|
+
and breaks in weird ways. This is the compiler version of that. You pay
|
|
153
|
+
the model cost once during exploration and get back tools like
|
|
154
|
+
`write_document(text)`. Calling them later costs no model tokens and takes
|
|
155
|
+
well under a second: on the author's Mac a read takes about 85 ms and a
|
|
156
|
+
write about 0.9 s, most of it spent waiting for the app to settle. When an
|
|
157
|
+
app update breaks a recorded action, a model is brought back in to fix it,
|
|
158
|
+
but the fix sits in quarantine until you approve it.
|
|
159
|
+
|
|
160
|
+
The top half of the diagram runs once per app and is where the model
|
|
161
|
+
earns its keep, with a policy screening every probe and a human approving
|
|
162
|
+
every tool. The bottom half is what you actually use day to day, and it
|
|
163
|
+
has no model in it at all. The pack file in the middle is the whole
|
|
164
|
+
interface between the two: plain JSON you can read, diff, and check into
|
|
165
|
+
a repo.
|
|
166
|
+
|
|
167
|
+
The accessibility layer has been in every Mac app for decades because
|
|
168
|
+
screen readers depend on it. It turns out it's also a pretty good API for
|
|
169
|
+
driving apps, if you're careful about how you find elements again later.
|
|
170
|
+
|
|
171
|
+
## How it compares
|
|
172
|
+
|
|
173
|
+
- **Screenshot computer use** (the computer-use features in Claude and
|
|
174
|
+
Codex, and most open-source agents): a model looks at the screen and
|
|
175
|
+
picks every action. It works on any app, and every step costs a model
|
|
176
|
+
call.
|
|
177
|
+
- **Accessibility MCP servers** such as
|
|
178
|
+
[Peekaboo](https://github.com/openclaw/Peekaboo) and
|
|
179
|
+
[MacOS-MCP](https://github.com/CursorTouch/MacOS-MCP): they hand the
|
|
180
|
+
accessibility tree and its actions to your client's model, which still
|
|
181
|
+
decides each click on every call. Nothing to record first, and nothing
|
|
182
|
+
fixed either.
|
|
183
|
+
- **AppleScript servers**: dependable where an app has a scripting
|
|
184
|
+
dictionary, which most Electron apps don't.
|
|
185
|
+
- **Ventriloquist**: the model works once, while you record. What gets
|
|
186
|
+
served is a fixed set of tools you approved, replayed with no model.
|
|
187
|
+
It covers fewer apps than any of the above, by design, and in exchange
|
|
188
|
+
calls are cheap, fast, and the same every time. The closest idea
|
|
189
|
+
elsewhere is [terminator](https://github.com/mediar-ai/terminator),
|
|
190
|
+
which records workflows as deterministic code on Windows.
|
|
191
|
+
|
|
192
|
+
## Make a pack for another app
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
vent doctor Spotify # does this app expose a usable tree?
|
|
196
|
+
vent explore Spotify --no-values # a model probes safely, the policy screens every action
|
|
197
|
+
vent compile Spotify # the model proposes tools; you approve each against its literal steps
|
|
198
|
+
vent run Spotify <tool> --arg k=v # call one tool directly
|
|
199
|
+
vent verify Spotify # dry-resolve anchors; promote quarantined heals
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
`vent explore`, `vent compile`, and `vent run --heal` call a model, and so
|
|
203
|
+
does `vent serve` if you start it with `VENT_HEAL=1` to let it heal broken
|
|
204
|
+
anchors. Nothing else does. The model comes from the Claude Code CLI if
|
|
205
|
+
it's installed and signed in, with no API key needed, or from the
|
|
206
|
+
Anthropic API: install with
|
|
207
|
+
`uv tool install --with anthropic git+https://github.com/ShaanGS/ventriloquist`
|
|
208
|
+
and set `ANTHROPIC_API_KEY`.
|
|
209
|
+
|
|
210
|
+
Packs you make live in `~/.ventriloquist/packs/` (set `VENT_HOME` to move
|
|
211
|
+
it), next to the traces `vent explore` records. A pack there replaces the
|
|
212
|
+
bundled pack for the same app, which is also where a healed bundled pack
|
|
213
|
+
ends up; delete your copy to go back to the bundled one.
|
|
214
|
+
|
|
215
|
+
## Limitations
|
|
216
|
+
|
|
217
|
+
This is an early release by one developer. What it doesn't do yet, or
|
|
218
|
+
does with caveats:
|
|
219
|
+
|
|
220
|
+
- **macOS only.** The whole approach rests on the macOS accessibility API.
|
|
221
|
+
- **The Accessibility grant is broad.** macOS gives it to the app that
|
|
222
|
+
starts vent (your terminal, Claude Desktop, Cursor), and anything that
|
|
223
|
+
app runs can then drive every app on your Mac. vent's own scoping
|
|
224
|
+
(named apps, a policy, risk levels) is self-discipline inside that
|
|
225
|
+
grant, not an OS boundary. SECURITY.md T10 has the details.
|
|
226
|
+
- **Packs are recorded on one machine.** Anchors held up under resizes,
|
|
227
|
+
full restarts, and a deliberately degraded environment on the author's
|
|
228
|
+
Mac, but nobody has measured them on a second Mac yet. A bundled pack
|
|
229
|
+
may refuse on yours and need re-recording.
|
|
230
|
+
- **The app needs a usable tree.** AppKit apps have one. Electron and
|
|
231
|
+
Chromium apps need `--force-renderer-accessibility`. Anything drawn on
|
|
232
|
+
a canvas (Figma's design surface, games) exposes its chrome and
|
|
233
|
+
nothing inside.
|
|
234
|
+
- **The app has to be running, with the window the tool expects.** The
|
|
235
|
+
Notes, VS Code, and Slack tools bring their app to the front while they
|
|
236
|
+
run and hand focus back afterwards. A locked screen makes every app
|
|
237
|
+
look empty.
|
|
238
|
+
- **Calls to one app run one at a time.** A GUI is one shared thing, so
|
|
239
|
+
parallel calls against the same app queue.
|
|
240
|
+
- **Preconditions are prose.** VS Code's `search_workspace` says to open
|
|
241
|
+
the Search view first, but nothing enforces it, and called cold it
|
|
242
|
+
fails. Stateful preconditions are planned.
|
|
243
|
+
- **High-risk tools are all or nothing.** They're refused unless the
|
|
244
|
+
server runs with `VENT_ALLOW_HIGH=1`; per-call confirmation isn't built
|
|
245
|
+
yet.
|
|
246
|
+
- **Exploring acts on your real app.** The explorer clicks and types
|
|
247
|
+
under a policy, and probes are restored, but an app that syncs every
|
|
248
|
+
keystroke may send a probe string before it's undone. Explore against
|
|
249
|
+
a scratch document or a test account, and use `--no-values` to keep
|
|
250
|
+
field contents away from the model.
|
|
251
|
+
- **`vent compile` replaces a pack instead of merging into it.**
|
|
252
|
+
- **Read tools hand app text to your client's model.** vent marks it as
|
|
253
|
+
untrusted data (SECURITY.md T12); whether the client respects that is
|
|
254
|
+
up to the client.
|
|
255
|
+
- **No system dialogs.** File pickers and save sheets are out of scope
|
|
256
|
+
for now.
|
|
257
|
+
|
|
258
|
+
## Anchor durability, measured
|
|
259
|
+
|
|
260
|
+
`vent harness <app>` records anchors for an app's elements, messes with
|
|
261
|
+
the app (a real window resize, and optionally a full quit and relaunch),
|
|
262
|
+
and then tries to resolve every anchor again. Survival only counts when
|
|
263
|
+
the element found is provably the one that was recorded. Binding a
|
|
264
|
+
lookalike counts as WRONG, which is the number that actually matters.
|
|
265
|
+
|
|
266
|
+
Numbers from this machine (macOS 26, small anchor sets, early days):
|
|
267
|
+
|
|
268
|
+
```
|
|
269
|
+
TextEdit: baseline 100%, resized 100%, 0 wrong
|
|
270
|
+
Finder: baseline 100%, resized 100%, 0 wrong
|
|
271
|
+
VS Code: baseline 95%, resized 87.5%, restart 92.5%, 0 wrong (40 anchors)
|
|
272
|
+
Slack: baseline 95%, resized 82.5%, restart 82.5%, 0 wrong (40 anchors)
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
Slack matters here more than the others. TextEdit and Finder have
|
|
276
|
+
AppleScript, VS Code has a CLI and an extension API, so the apps that
|
|
277
|
+
justify this project are the ones with no scripting surface at all, and
|
|
278
|
+
Slack is the first of those measured. Its tree has no identifiers
|
|
279
|
+
anywhere, so anchors ride entirely on labels and ancestor chains, and
|
|
280
|
+
they still come back with zero wrong bindings across resize and a full
|
|
281
|
+
restart. Every non-survivor is an ambiguity refusal.
|
|
282
|
+
|
|
283
|
+
Figma desktop is the deliberate worst case and it behaves like one: the
|
|
284
|
+
app shell (tabs, navigation, buttons) publishes a usable tree, and the
|
|
285
|
+
design canvas publishes nothing, because it's rendered pixels. That's
|
|
286
|
+
the honest boundary of the accessibility thesis. Apps whose content is
|
|
287
|
+
a custom-drawn canvas keep their chrome scriptable and their content
|
|
288
|
+
opaque.
|
|
289
|
+
|
|
290
|
+
The VS Code restart round quits and relaunches the app completely, and 37
|
|
291
|
+
of 40 anchors still bind to provably the same element afterward. The ones
|
|
292
|
+
that don't survive are refusals, not mistakes: Chromium trees are full of
|
|
293
|
+
unlabeled twin groups, and when two candidates score too close together
|
|
294
|
+
the resolver refuses to pick one. The VS Code pack's own curated anchors
|
|
295
|
+
went 6 for 6 across view switches, a resize, and an actual app update.
|
|
296
|
+
|
|
297
|
+
Per-anchor survival flatters what a caller actually experiences, because
|
|
298
|
+
a tool touches several anchors plus settling and verification, so there
|
|
299
|
+
is a second harness mode that runs whole tools end to end.
|
|
300
|
+
`vent harness <app> --tools` executes every tool in the app's pack
|
|
301
|
+
repeatedly, resizing the window every other cycle:
|
|
302
|
+
|
|
303
|
+
```
|
|
304
|
+
TextEdit: 8/8 calls passed (4 cycles)
|
|
305
|
+
VS Code: 12/16 calls passed (4 cycles, every failure the same tool)
|
|
306
|
+
Slack: 6/8 calls passed (4 cycles, both failures ambiguity refusals)
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
The Slack pack is thin on purpose (navigation and a parameterized
|
|
310
|
+
sidebar filter) but it is compiled capability, not just anchor survival.
|
|
311
|
+
Its two failures happen when the resized window reflows the nav rail and
|
|
312
|
+
the Home tab gains a lookalike; the resolver refuses to pick between
|
|
313
|
+
them, which is a safe failure and still a failure. Nothing wrong was
|
|
314
|
+
pressed in any cycle on any app.
|
|
315
|
+
|
|
316
|
+
The VS Code failures are all `search_workspace` called cold. Its
|
|
317
|
+
description says to open the Search view first, but that dependency is
|
|
318
|
+
prose, not a machine-checked precondition, so a caller who ignores it
|
|
319
|
+
fails and so does this harness, on purpose. The anchors were fine in
|
|
320
|
+
every failing call. That gap between 87.5% per-anchor and 75% per-tool
|
|
321
|
+
is the honest cost of state in a GUI, and stateful preconditions are the
|
|
322
|
+
planned fix.
|
|
323
|
+
|
|
324
|
+
There is also a first portability data point, from a deliberately
|
|
325
|
+
degraded environment rather than a second machine: the VS Code pack was
|
|
326
|
+
re-verified after collapsing the sidebar, closing every file and folder,
|
|
327
|
+
switching theme, disabling extensions, and resizing the window. The
|
|
328
|
+
three activity bar anchors resolved exactly, unmoved by the extension
|
|
329
|
+
removals that changed their sibling set. The two view-content anchors
|
|
330
|
+
whose views no longer existed failed the right way, one by refusing
|
|
331
|
+
outright, and one by binding a same-role neighbor, an open editor pane,
|
|
332
|
+
which the semantic drift check flagged. That near-miss became a runtime
|
|
333
|
+
rule: an element wearing a label its anchor has never been recorded
|
|
334
|
+
under is never acted on, so the call that would have typed into the open
|
|
335
|
+
editor now refuses and names the element it declined to touch. What this
|
|
336
|
+
run does not show is cross-machine portability; it was the same machine,
|
|
337
|
+
account, and app install. `vent verify <app> --report out.json` exists
|
|
338
|
+
for the real test: it produces a small JSON resolution table with no
|
|
339
|
+
model calls and no app content beyond each element's live label, meant
|
|
340
|
+
to be run on someone else's Mac and sent back.
|
|
341
|
+
|
|
342
|
+
Some field notes that ended up shaping the code: macOS AX trees can be
|
|
343
|
+
cyclic (a wedged TextEdit once returned the app element as its own child,
|
|
344
|
+
so every walk carries a cycle guard), apps relaunched from a background
|
|
345
|
+
process serve junk menubar-only trees until they're genuinely foregrounded,
|
|
346
|
+
a pending permission dialog silently degrades AX for every app launched
|
|
347
|
+
after it, and a locked screen makes every app read as empty while the
|
|
348
|
+
permission check still passes.
|
|
349
|
+
|
|
350
|
+
## Design
|
|
351
|
+
|
|
352
|
+
Read [docs/ARCHITECTURE.md](https://github.com/ShaanGS/ventriloquist/blob/main/docs/ARCHITECTURE.md)
|
|
353
|
+
before the code, and [docs/SECURITY.md](https://github.com/ShaanGS/ventriloquist/blob/main/docs/SECURITY.md)
|
|
354
|
+
for the threat model. Both went through adversarial review before
|
|
355
|
+
implementation and got rewritten from the findings, and each phase since
|
|
356
|
+
has gotten the same treatment.
|
|
357
|
+
[docs/HANDOFF.md](https://github.com/ShaanGS/ventriloquist/blob/main/docs/HANDOFF.md)
|
|
358
|
+
is the current state and what's next.
|
|
359
|
+
|
|
360
|
+
## Contributing
|
|
361
|
+
|
|
362
|
+
Bug reports from Macs other than the author's are the most useful thing
|
|
363
|
+
right now, especially `vent verify <app> --report out.json` output for a
|
|
364
|
+
bundled pack. To work on the code:
|
|
365
|
+
|
|
366
|
+
```bash
|
|
367
|
+
git clone https://github.com/ShaanGS/ventriloquist && cd ventriloquist
|
|
368
|
+
python3 -m venv .venv && .venv/bin/pip install -e . pytest
|
|
369
|
+
.venv/bin/python -m pytest tests/
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
The test suite needs no Mac permissions, no live apps, and no model
|
|
373
|
+
credentials, and runs in CI on Linux and macOS.
|
|
374
|
+
[CONTRIBUTING.md](https://github.com/ShaanGS/ventriloquist/blob/main/CONTRIBUTING.md)
|
|
375
|
+
has the rules that are load-bearing.
|
|
376
|
+
|
|
377
|
+
## License
|
|
378
|
+
|
|
379
|
+
MIT
|