bad-ass-mcp 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bad_ass_mcp-1.4.0/.github/FUNDING.yml +1 -0
- bad_ass_mcp-1.4.0/.github/workflows/ci.yml +33 -0
- bad_ass_mcp-1.4.0/.github/workflows/release.yml +118 -0
- bad_ass_mcp-1.4.0/.gitignore +12 -0
- bad_ass_mcp-1.4.0/.pre-commit-config.yaml +7 -0
- bad_ass_mcp-1.4.0/LICENSE +21 -0
- bad_ass_mcp-1.4.0/PKG-INFO +231 -0
- bad_ass_mcp-1.4.0/README.md +193 -0
- bad_ass_mcp-1.4.0/docs/demo.gif +0 -0
- bad_ass_mcp-1.4.0/pyproject.toml +58 -0
- bad_ass_mcp-1.4.0/scripts/demo_doc.md +26 -0
- bad_ass_mcp-1.4.0/scripts/demo_sequence.py +98 -0
- bad_ass_mcp-1.4.0/scripts/record.sh +4 -0
- bad_ass_mcp-1.4.0/scripts/record_demo.py +101 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/__init__.py +0 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/backend/__init__.py +0 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/backend/base.py +282 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/backend/linux.py +1079 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/backend/macos.py +904 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/backend/windows.py +955 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/server.py +413 -0
- bad_ass_mcp-1.4.0/src/bad_ass_mcp/types.py +49 -0
- bad_ass_mcp-1.4.0/tests/__init__.py +0 -0
- bad_ass_mcp-1.4.0/tests/fake_backend.py +152 -0
- bad_ass_mcp-1.4.0/tests/test_backend_contract.py +338 -0
- bad_ass_mcp-1.4.0/tests/test_linux_internals.py +402 -0
- bad_ass_mcp-1.4.0/tests/test_macos_internals.py +662 -0
- bad_ass_mcp-1.4.0/tests/test_prune_tree.py +125 -0
- bad_ass_mcp-1.4.0/tests/test_serialise_tree.py +118 -0
- bad_ass_mcp-1.4.0/tests/test_windows_internals.py +244 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
github: jabberwock
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
runs-on: ubuntu-26.04
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v7
|
|
13
|
+
- uses: actions/setup-python@v7
|
|
14
|
+
with:
|
|
15
|
+
python-version: "3.11"
|
|
16
|
+
- run: python -m pip install -U "pip>=26.2.2" "certifi>=2026.10.19"
|
|
17
|
+
- run: pip install ruff
|
|
18
|
+
- run: ruff check .
|
|
19
|
+
- run: ruff format --check .
|
|
20
|
+
|
|
21
|
+
test:
|
|
22
|
+
runs-on: ubuntu-26.04
|
|
23
|
+
strategy:
|
|
24
|
+
matrix:
|
|
25
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v7
|
|
28
|
+
- uses: actions/setup-python@v7
|
|
29
|
+
with:
|
|
30
|
+
python-version: ${{ matrix.python-version }}
|
|
31
|
+
- run: python -m pip install -U "pip>=26.2.2" "certifi>=2026.10.19"
|
|
32
|
+
- run: pip install -e ".[dev]"
|
|
33
|
+
- run: pytest
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
# ── Per-platform PyInstaller bundles ─────────────────────────────────
|
|
10
|
+
bundle:
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
include:
|
|
14
|
+
- os: macos-latest
|
|
15
|
+
extra: macos
|
|
16
|
+
name: macos
|
|
17
|
+
hidden: bad_ass_mcp.backend.macos
|
|
18
|
+
- os: windows-latest
|
|
19
|
+
extra: windows
|
|
20
|
+
name: windows
|
|
21
|
+
hidden: bad_ass_mcp.backend.windows
|
|
22
|
+
|
|
23
|
+
runs-on: ${{ matrix.os }}
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v7
|
|
27
|
+
with:
|
|
28
|
+
fetch-depth: 0 # needed for hatch-vcs tag resolution
|
|
29
|
+
|
|
30
|
+
- uses: actions/setup-python@v7
|
|
31
|
+
with:
|
|
32
|
+
python-version: "3.11"
|
|
33
|
+
|
|
34
|
+
# emSign/eMudhra CA incident (Oct 2026): old pip trusts the pulled
|
|
35
|
+
# roots via its vendored certifi, so upgrade before installing anything.
|
|
36
|
+
- name: Upgrade pip + certifi
|
|
37
|
+
run: python -m pip install -U "pip>=26.2.2" "certifi>=2026.10.19"
|
|
38
|
+
|
|
39
|
+
- name: Install
|
|
40
|
+
run: pip install pyinstaller ".[${{ matrix.extra }}]"
|
|
41
|
+
|
|
42
|
+
- name: Build standalone
|
|
43
|
+
shell: bash
|
|
44
|
+
run: |
|
|
45
|
+
pyinstaller \
|
|
46
|
+
--onedir \
|
|
47
|
+
--name bad-ass-mcp \
|
|
48
|
+
--collect-all anyio \
|
|
49
|
+
--hidden-import ${{ matrix.hidden }} \
|
|
50
|
+
src/bad_ass_mcp/server.py
|
|
51
|
+
|
|
52
|
+
- name: Zip (Unix)
|
|
53
|
+
if: runner.os != 'Windows'
|
|
54
|
+
run: cd dist && zip -r ../bad-ass-mcp-${{ matrix.name }}.zip bad-ass-mcp/
|
|
55
|
+
|
|
56
|
+
- name: Zip (Windows)
|
|
57
|
+
if: runner.os == 'Windows'
|
|
58
|
+
shell: pwsh
|
|
59
|
+
run: Compress-Archive -Path dist/bad-ass-mcp -DestinationPath bad-ass-mcp-${{ matrix.name }}.zip
|
|
60
|
+
|
|
61
|
+
- uses: actions/upload-artifact@v7
|
|
62
|
+
with:
|
|
63
|
+
name: bundle-${{ matrix.name }}
|
|
64
|
+
path: bad-ass-mcp-${{ matrix.name }}.zip
|
|
65
|
+
|
|
66
|
+
# ── Publish sdist + wheel + bundles to GitHub Release ────────────────
|
|
67
|
+
release:
|
|
68
|
+
needs: bundle
|
|
69
|
+
runs-on: ubuntu-26.04
|
|
70
|
+
permissions:
|
|
71
|
+
contents: write
|
|
72
|
+
|
|
73
|
+
steps:
|
|
74
|
+
- uses: actions/checkout@v7
|
|
75
|
+
with:
|
|
76
|
+
fetch-depth: 0
|
|
77
|
+
|
|
78
|
+
- uses: actions/setup-python@v7
|
|
79
|
+
with:
|
|
80
|
+
python-version: "3.11"
|
|
81
|
+
|
|
82
|
+
- name: Upgrade pip + certifi
|
|
83
|
+
run: python -m pip install -U "pip>=26.2.2" "certifi>=2026.10.19"
|
|
84
|
+
|
|
85
|
+
- name: Build sdist + wheel
|
|
86
|
+
run: pip install hatch hatch-vcs && hatch build
|
|
87
|
+
|
|
88
|
+
- uses: actions/upload-artifact@v7
|
|
89
|
+
with:
|
|
90
|
+
name: python-dist
|
|
91
|
+
path: dist/
|
|
92
|
+
|
|
93
|
+
- uses: actions/download-artifact@v8
|
|
94
|
+
with:
|
|
95
|
+
path: bundles
|
|
96
|
+
merge-multiple: true
|
|
97
|
+
|
|
98
|
+
- uses: softprops/action-gh-release@v2
|
|
99
|
+
with:
|
|
100
|
+
files: |
|
|
101
|
+
dist/*
|
|
102
|
+
bundles/*.zip
|
|
103
|
+
|
|
104
|
+
# ── Publish sdist + wheel to PyPI (trusted publishing, no token) ─────
|
|
105
|
+
pypi:
|
|
106
|
+
needs: release
|
|
107
|
+
runs-on: ubuntu-26.04
|
|
108
|
+
environment: pypi
|
|
109
|
+
permissions:
|
|
110
|
+
id-token: write
|
|
111
|
+
|
|
112
|
+
steps:
|
|
113
|
+
- uses: actions/download-artifact@v8
|
|
114
|
+
with:
|
|
115
|
+
name: python-dist
|
|
116
|
+
path: dist
|
|
117
|
+
|
|
118
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Michael Behan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: bad-ass-mcp
|
|
3
|
+
Version: 1.4.0
|
|
4
|
+
Summary: Cross-platform desktop GUI automation MCP — accessibility tree first, vision as last resort
|
|
5
|
+
Project-URL: Homepage, https://github.com/HoldMyBeer-gg/bad-ass-mcp
|
|
6
|
+
Project-URL: Repository, https://github.com/HoldMyBeer-gg/bad-ass-mcp
|
|
7
|
+
Project-URL: Issues, https://github.com/HoldMyBeer-gg/bad-ass-mcp/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/HoldMyBeer-gg/bad-ass-mcp/releases
|
|
9
|
+
Author: Michael Behan
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: a11y,accessibility,automation,claude,desktop,gui,mcp
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
18
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Desktop Environment
|
|
24
|
+
Classifier: Topic :: Software Development :: Testing
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: mcp<2,>=1.0.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.9; extra == 'dev'
|
|
31
|
+
Provides-Extra: macos
|
|
32
|
+
Requires-Dist: pyobjc-framework-applicationservices>=9.0; extra == 'macos'
|
|
33
|
+
Requires-Dist: pyobjc-framework-cocoa>=9.0; extra == 'macos'
|
|
34
|
+
Requires-Dist: pyobjc-framework-quartz>=9.0; extra == 'macos'
|
|
35
|
+
Provides-Extra: windows
|
|
36
|
+
Requires-Dist: comtypes>=1.4; extra == 'windows'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# bad-ass-mcp
|
|
40
|
+
|
|
41
|
+
*Every other tool needs a browser or a vision model. This one reads what the OS already knows.*
|
|
42
|
+
|
|
43
|
+
A cross-platform MCP server for desktop GUI automation — driven by accessibility APIs, not screenshots.
|
|
44
|
+
|
|
45
|
+
Works with Claude, Codex, Gemini, Ollama, or any OpenAI-compatible client that supports MCP.
|
|
46
|
+
|
|
47
|
+

|
|
48
|
+
*bad-ass-mcp recording itself interacting with [MarkTheCrab](https://github.com/HoldMyBeer-gg/MarkTheCrab)*
|
|
49
|
+
|
|
50
|
+
## Why
|
|
51
|
+
|
|
52
|
+
Most AI desktop automation tools work by taking screenshots and asking a vision model "what do you see?" That's slow, expensive, and fragile.
|
|
53
|
+
|
|
54
|
+
Every major OS exposes a structured accessibility tree — the same one used by screen readers — that describes every button, text field, combo box, and menu in every running application. **bad-ass-mcp** speaks that language directly.
|
|
55
|
+
|
|
56
|
+
- **macOS**: AXUIElement via PyObjC
|
|
57
|
+
- **Linux**: AT-SPI2 via `gi.repository.Atspi`
|
|
58
|
+
- **Windows**: UI Automation via `comtypes`
|
|
59
|
+
|
|
60
|
+
Actions fire on control objects directly, so the target window doesn't need focus. The user can keep working while automation runs in the background.
|
|
61
|
+
|
|
62
|
+
## Tools
|
|
63
|
+
|
|
64
|
+
| Tool | Description |
|
|
65
|
+
|------|-------------|
|
|
66
|
+
| `list_windows` | List all visible application windows |
|
|
67
|
+
| `get_tree` | Full accessibility tree for a window as nested JSON |
|
|
68
|
+
| `find_elements` | Find elements by role and/or name |
|
|
69
|
+
| `click` | Click / invoke an element (foreground-independent) |
|
|
70
|
+
| `click_at` | Click at absolute screen coordinates — fallback for webview/canvas UI |
|
|
71
|
+
| `type_text` | Type text into a field (SetValue → EditableText → key injection) |
|
|
72
|
+
| `select_option` | Select an option in a combo box or list |
|
|
73
|
+
| `get_value` | Get the current value or text of an element |
|
|
74
|
+
| `wait_for_window` | Wait until a window matching a pattern appears |
|
|
75
|
+
| `wait_for_element` | Wait until an element exists and optionally has a given state |
|
|
76
|
+
| `screenshot` | Capture a PNG (write to `output_path` or return base64) |
|
|
77
|
+
| `start_recording` | Start recording the screen (or a specific window) |
|
|
78
|
+
| `stop_recording` | Stop recording and export as a GIF |
|
|
79
|
+
| `learn_layout` | Resolve semantic names → live handle IDs (one call per session) |
|
|
80
|
+
| `run_sequence` | Execute a list of actions server-side — no per-action round-trips |
|
|
81
|
+
|
|
82
|
+
## Webviews: waking lazy accessibility trees
|
|
83
|
+
|
|
84
|
+
Chromium-based apps (Electron, CEF, and every Chromium browser) have a
|
|
85
|
+
full accessibility tree — they just don't *build* it until an assistive
|
|
86
|
+
technology announces itself. Until then the window looks canvas-only:
|
|
87
|
+
`get_tree` returns nothing and screenshot-clicking is the only option.
|
|
88
|
+
|
|
89
|
+
**bad-ass-mcp announces itself.** When a window's a11y tree comes back
|
|
90
|
+
empty and the process looks like a webview, the backend performs the
|
|
91
|
+
platform's "a screen reader is here" handshake, waits a beat, and
|
|
92
|
+
re-probes before giving up:
|
|
93
|
+
|
|
94
|
+
- **Linux**: sets `org.a11y.Status.ScreenReaderEnabled` on the session
|
|
95
|
+
D-Bus **at server start** so Chromium/Electron apps launched
|
|
96
|
+
afterwards enable accessibility from birth. Windows that stay hollow
|
|
97
|
+
(an application+frame whose phantom child fetches as `None`) are
|
|
98
|
+
detected via the AT-SPI toolkit name and reported
|
|
99
|
+
`accessible: false`: the app either launched before the flag went up
|
|
100
|
+
or opts out on its own. Restarting the app with the server running
|
|
101
|
+
should pick the flag up per Chromium's documented behavior, but note
|
|
102
|
+
that is not yet verified here: the only Linux app live-tested so far
|
|
103
|
+
(flatpak Vivaldi) ignores the flag entirely, its own first-run
|
|
104
|
+
assistive-technology toggle included, and only produces a tree when
|
|
105
|
+
launched with `--force-renderer-accessibility`.
|
|
106
|
+
- **macOS**: sets `AXManualAccessibility` on the app (Electron's
|
|
107
|
+
documented switch), falling back to `AXEnhancedUserInterface` (what
|
|
108
|
+
VoiceOver sets, watched by plain Chrome/CEF). The attribute set only
|
|
109
|
+
succeeds on apps that support it, so it's a free no-op elsewhere.
|
|
110
|
+
- **Windows**: sets the system `SPI_SETSCREENREADERRUNNING` flag, then
|
|
111
|
+
queries the `Chrome_RenderWidgetHostHWND` child window — Chromium
|
|
112
|
+
hangs its DOM tree off that renderer child, not the top-level HWND
|
|
113
|
+
(which only shows empty panes). `get_tree` / `find_elements` are
|
|
114
|
+
routed through the renderer child automatically.
|
|
115
|
+
|
|
116
|
+
One wake attempt per process per server run. If `list_windows` still
|
|
117
|
+
reports `accessible: false` after this, the window is genuinely
|
|
118
|
+
canvas-only (custom OpenGL/Vulkan surface, immediate-mode toolkit
|
|
119
|
+
without AccessKit) — fall back to `screenshot` + `click_at`.
|
|
120
|
+
|
|
121
|
+
The tradeoff: a woken app spends some memory/CPU maintaining its tree,
|
|
122
|
+
which is exactly why Chromium lazy-loads it. The wake targets the app
|
|
123
|
+
you're automating, not the whole desktop (the Linux flag is technically
|
|
124
|
+
session-global, but only a11y-aware apps respond, and only on demand).
|
|
125
|
+
|
|
126
|
+
## Installation
|
|
127
|
+
|
|
128
|
+
**Requirements**: Python 3.11+, `ffmpeg` + `gifsicle` for recording (optional)
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# macOS
|
|
132
|
+
pip install 'bad-ass-mcp[macos]'
|
|
133
|
+
|
|
134
|
+
# Linux — install system PyGObject + AT-SPI bindings first, then:
|
|
135
|
+
pip install bad-ass-mcp
|
|
136
|
+
|
|
137
|
+
# Windows
|
|
138
|
+
pip install 'bad-ass-mcp[windows]'
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
**Linux**: PyGObject and AT-SPI are not on PyPI — install them from your distro:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
# Debian/Ubuntu
|
|
145
|
+
sudo apt install python3-gi gir1.2-atspi-2.0 at-spi2-core
|
|
146
|
+
|
|
147
|
+
# Arch
|
|
148
|
+
sudo pacman -S python-gobject at-spi2-core
|
|
149
|
+
|
|
150
|
+
# Fedora
|
|
151
|
+
sudo dnf install python3-gobject at-spi2-core
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Then make sure AT-SPI is enabled (most desktop environments — GNOME, KDE, XFCE — have it on by default):
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
# Check
|
|
158
|
+
gsettings get org.gnome.desktop.interface toolkit-accessibility
|
|
159
|
+
|
|
160
|
+
# Enable
|
|
161
|
+
gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
If you're using a venv, create it with `--system-site-packages` so it can see the distro-installed `gi`:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
python -m venv --system-site-packages venv
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
**Windows**: Elevated (admin) applications can only be automated from an elevated Python process. No special permissions are needed for normal apps.
|
|
171
|
+
|
|
172
|
+
### Register with Claude Code
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
claude mcp add bad-ass-mcp --scope user -- bad-ass-mcp
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Or manually in `~/.claude.json`:
|
|
179
|
+
|
|
180
|
+
```json
|
|
181
|
+
{
|
|
182
|
+
"mcpServers": {
|
|
183
|
+
"bad-ass-mcp": {
|
|
184
|
+
"type": "stdio",
|
|
185
|
+
"command": "bad-ass-mcp"
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## Usage
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
list the windows on screen
|
|
195
|
+
→ [{ "id": "12345", "name": "Firefox", ... }]
|
|
196
|
+
|
|
197
|
+
find the search box in window 12345 and type "hello"
|
|
198
|
+
→ find_elements(window_id="12345", role="entry")
|
|
199
|
+
→ type_text(handle_id="...", text="hello")
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
Electron/CEF apps get their lazy accessibility tree woken automatically (see [Webviews](#webviews-waking-lazy-accessibility-trees) above). For apps that still don't expose a clean tree, `type_text` falls back to platform-native key injection automatically (AT-SPI on Linux, Quartz events on macOS, `PostMessage` on Windows).
|
|
203
|
+
|
|
204
|
+
## Architecture
|
|
205
|
+
|
|
206
|
+
```
|
|
207
|
+
bad_ass_mcp/
|
|
208
|
+
├── server.py # FastMCP tool definitions
|
|
209
|
+
├── types.py # WindowInfo, ElementHandle, ActionResult, StaleHandleError
|
|
210
|
+
└── backend/
|
|
211
|
+
├── base.py # Abstract DesktopBackend interface
|
|
212
|
+
├── linux.py # AT-SPI2 + xdotool
|
|
213
|
+
├── macos.py # AXUIElement + Quartz
|
|
214
|
+
└── windows.py # UI Automation + ctypes Win32
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
The abstract backend interface means adding a new platform is just implementing one class. The full contract-test suite runs against every backend via `FakeBackend`.
|
|
218
|
+
|
|
219
|
+
## Releasing
|
|
220
|
+
|
|
221
|
+
Tag a version and the release workflow builds the sdist and wheel, publishes them
|
|
222
|
+
to PyPI via trusted publishing, and attaches them (plus the PyInstaller bundles)
|
|
223
|
+
to the GitHub release:
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
git tag v1.4.0 && git push --tags
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
## License
|
|
230
|
+
|
|
231
|
+
MIT
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# bad-ass-mcp
|
|
2
|
+
|
|
3
|
+
*Every other tool needs a browser or a vision model. This one reads what the OS already knows.*
|
|
4
|
+
|
|
5
|
+
A cross-platform MCP server for desktop GUI automation — driven by accessibility APIs, not screenshots.
|
|
6
|
+
|
|
7
|
+
Works with Claude, Codex, Gemini, Ollama, or any OpenAI-compatible client that supports MCP.
|
|
8
|
+
|
|
9
|
+

|
|
10
|
+
*bad-ass-mcp recording itself interacting with [MarkTheCrab](https://github.com/HoldMyBeer-gg/MarkTheCrab)*
|
|
11
|
+
|
|
12
|
+
## Why
|
|
13
|
+
|
|
14
|
+
Most AI desktop automation tools work by taking screenshots and asking a vision model "what do you see?" That's slow, expensive, and fragile.
|
|
15
|
+
|
|
16
|
+
Every major OS exposes a structured accessibility tree — the same one used by screen readers — that describes every button, text field, combo box, and menu in every running application. **bad-ass-mcp** speaks that language directly.
|
|
17
|
+
|
|
18
|
+
- **macOS**: AXUIElement via PyObjC
|
|
19
|
+
- **Linux**: AT-SPI2 via `gi.repository.Atspi`
|
|
20
|
+
- **Windows**: UI Automation via `comtypes`
|
|
21
|
+
|
|
22
|
+
Actions fire on control objects directly, so the target window doesn't need focus. The user can keep working while automation runs in the background.
|
|
23
|
+
|
|
24
|
+
## Tools
|
|
25
|
+
|
|
26
|
+
| Tool | Description |
|
|
27
|
+
|------|-------------|
|
|
28
|
+
| `list_windows` | List all visible application windows |
|
|
29
|
+
| `get_tree` | Full accessibility tree for a window as nested JSON |
|
|
30
|
+
| `find_elements` | Find elements by role and/or name |
|
|
31
|
+
| `click` | Click / invoke an element (foreground-independent) |
|
|
32
|
+
| `click_at` | Click at absolute screen coordinates — fallback for webview/canvas UI |
|
|
33
|
+
| `type_text` | Type text into a field (SetValue → EditableText → key injection) |
|
|
34
|
+
| `select_option` | Select an option in a combo box or list |
|
|
35
|
+
| `get_value` | Get the current value or text of an element |
|
|
36
|
+
| `wait_for_window` | Wait until a window matching a pattern appears |
|
|
37
|
+
| `wait_for_element` | Wait until an element exists and optionally has a given state |
|
|
38
|
+
| `screenshot` | Capture a PNG (write to `output_path` or return base64) |
|
|
39
|
+
| `start_recording` | Start recording the screen (or a specific window) |
|
|
40
|
+
| `stop_recording` | Stop recording and export as a GIF |
|
|
41
|
+
| `learn_layout` | Resolve semantic names → live handle IDs (one call per session) |
|
|
42
|
+
| `run_sequence` | Execute a list of actions server-side — no per-action round-trips |
|
|
43
|
+
|
|
44
|
+
## Webviews: waking lazy accessibility trees
|
|
45
|
+
|
|
46
|
+
Chromium-based apps (Electron, CEF, and every Chromium browser) have a
|
|
47
|
+
full accessibility tree — they just don't *build* it until an assistive
|
|
48
|
+
technology announces itself. Until then the window looks canvas-only:
|
|
49
|
+
`get_tree` returns nothing and screenshot-clicking is the only option.
|
|
50
|
+
|
|
51
|
+
**bad-ass-mcp announces itself.** When a window's a11y tree comes back
|
|
52
|
+
empty and the process looks like a webview, the backend performs the
|
|
53
|
+
platform's "a screen reader is here" handshake, waits a beat, and
|
|
54
|
+
re-probes before giving up:
|
|
55
|
+
|
|
56
|
+
- **Linux**: sets `org.a11y.Status.ScreenReaderEnabled` on the session
|
|
57
|
+
D-Bus **at server start** so Chromium/Electron apps launched
|
|
58
|
+
afterwards enable accessibility from birth. Windows that stay hollow
|
|
59
|
+
(an application+frame whose phantom child fetches as `None`) are
|
|
60
|
+
detected via the AT-SPI toolkit name and reported
|
|
61
|
+
`accessible: false`: the app either launched before the flag went up
|
|
62
|
+
or opts out on its own. Restarting the app with the server running
|
|
63
|
+
should pick the flag up per Chromium's documented behavior, but note
|
|
64
|
+
that is not yet verified here: the only Linux app live-tested so far
|
|
65
|
+
(flatpak Vivaldi) ignores the flag entirely, its own first-run
|
|
66
|
+
assistive-technology toggle included, and only produces a tree when
|
|
67
|
+
launched with `--force-renderer-accessibility`.
|
|
68
|
+
- **macOS**: sets `AXManualAccessibility` on the app (Electron's
|
|
69
|
+
documented switch), falling back to `AXEnhancedUserInterface` (what
|
|
70
|
+
VoiceOver sets, watched by plain Chrome/CEF). The attribute set only
|
|
71
|
+
succeeds on apps that support it, so it's a free no-op elsewhere.
|
|
72
|
+
- **Windows**: sets the system `SPI_SETSCREENREADERRUNNING` flag, then
|
|
73
|
+
queries the `Chrome_RenderWidgetHostHWND` child window — Chromium
|
|
74
|
+
hangs its DOM tree off that renderer child, not the top-level HWND
|
|
75
|
+
(which only shows empty panes). `get_tree` / `find_elements` are
|
|
76
|
+
routed through the renderer child automatically.
|
|
77
|
+
|
|
78
|
+
One wake attempt per process per server run. If `list_windows` still
|
|
79
|
+
reports `accessible: false` after this, the window is genuinely
|
|
80
|
+
canvas-only (custom OpenGL/Vulkan surface, immediate-mode toolkit
|
|
81
|
+
without AccessKit) — fall back to `screenshot` + `click_at`.
|
|
82
|
+
|
|
83
|
+
The tradeoff: a woken app spends some memory/CPU maintaining its tree,
|
|
84
|
+
which is exactly why Chromium lazy-loads it. The wake targets the app
|
|
85
|
+
you're automating, not the whole desktop (the Linux flag is technically
|
|
86
|
+
session-global, but only a11y-aware apps respond, and only on demand).
|
|
87
|
+
|
|
88
|
+
## Installation
|
|
89
|
+
|
|
90
|
+
**Requirements**: Python 3.11+, `ffmpeg` + `gifsicle` for recording (optional)
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
# macOS
|
|
94
|
+
pip install 'bad-ass-mcp[macos]'
|
|
95
|
+
|
|
96
|
+
# Linux — install system PyGObject + AT-SPI bindings first, then:
|
|
97
|
+
pip install bad-ass-mcp
|
|
98
|
+
|
|
99
|
+
# Windows
|
|
100
|
+
pip install 'bad-ass-mcp[windows]'
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
**Linux**: PyGObject and AT-SPI are not on PyPI — install them from your distro:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
# Debian/Ubuntu
|
|
107
|
+
sudo apt install python3-gi gir1.2-atspi-2.0 at-spi2-core
|
|
108
|
+
|
|
109
|
+
# Arch
|
|
110
|
+
sudo pacman -S python-gobject at-spi2-core
|
|
111
|
+
|
|
112
|
+
# Fedora
|
|
113
|
+
sudo dnf install python3-gobject at-spi2-core
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Then make sure AT-SPI is enabled (most desktop environments — GNOME, KDE, XFCE — have it on by default):
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
# Check
|
|
120
|
+
gsettings get org.gnome.desktop.interface toolkit-accessibility
|
|
121
|
+
|
|
122
|
+
# Enable
|
|
123
|
+
gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
If you're using a venv, create it with `--system-site-packages` so it can see the distro-installed `gi`:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
python -m venv --system-site-packages venv
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**Windows**: Elevated (admin) applications can only be automated from an elevated Python process. No special permissions are needed for normal apps.
|
|
133
|
+
|
|
134
|
+
### Register with Claude Code
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
claude mcp add bad-ass-mcp --scope user -- bad-ass-mcp
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Or manually in `~/.claude.json`:
|
|
141
|
+
|
|
142
|
+
```json
|
|
143
|
+
{
|
|
144
|
+
"mcpServers": {
|
|
145
|
+
"bad-ass-mcp": {
|
|
146
|
+
"type": "stdio",
|
|
147
|
+
"command": "bad-ass-mcp"
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Usage
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
list the windows on screen
|
|
157
|
+
→ [{ "id": "12345", "name": "Firefox", ... }]
|
|
158
|
+
|
|
159
|
+
find the search box in window 12345 and type "hello"
|
|
160
|
+
→ find_elements(window_id="12345", role="entry")
|
|
161
|
+
→ type_text(handle_id="...", text="hello")
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Electron/CEF apps get their lazy accessibility tree woken automatically (see [Webviews](#webviews-waking-lazy-accessibility-trees) above). For apps that still don't expose a clean tree, `type_text` falls back to platform-native key injection automatically (AT-SPI on Linux, Quartz events on macOS, `PostMessage` on Windows).
|
|
165
|
+
|
|
166
|
+
## Architecture
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
bad_ass_mcp/
|
|
170
|
+
├── server.py # FastMCP tool definitions
|
|
171
|
+
├── types.py # WindowInfo, ElementHandle, ActionResult, StaleHandleError
|
|
172
|
+
└── backend/
|
|
173
|
+
├── base.py # Abstract DesktopBackend interface
|
|
174
|
+
├── linux.py # AT-SPI2 + xdotool
|
|
175
|
+
├── macos.py # AXUIElement + Quartz
|
|
176
|
+
└── windows.py # UI Automation + ctypes Win32
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
The abstract backend interface means adding a new platform is just implementing one class. The full contract-test suite runs against every backend via `FakeBackend`.
|
|
180
|
+
|
|
181
|
+
## Releasing
|
|
182
|
+
|
|
183
|
+
Tag a version and the release workflow builds the sdist and wheel, publishes them
|
|
184
|
+
to PyPI via trusted publishing, and attaches them (plus the PyInstaller bundles)
|
|
185
|
+
to the GitHub release:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
git tag v1.4.0 && git push --tags
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## License
|
|
192
|
+
|
|
193
|
+
MIT
|
|
Binary file
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "bad-ass-mcp"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Cross-platform desktop GUI automation MCP — accessibility tree first, vision as last resort"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Michael Behan" }]
|
|
13
|
+
keywords = ["mcp", "accessibility", "automation", "desktop", "gui", "a11y", "claude"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: MacOS",
|
|
19
|
+
"Operating System :: Microsoft :: Windows",
|
|
20
|
+
"Operating System :: POSIX :: Linux",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Software Development :: Testing",
|
|
26
|
+
"Topic :: Desktop Environment",
|
|
27
|
+
]
|
|
28
|
+
requires-python = ">=3.11"
|
|
29
|
+
dependencies = [
|
|
30
|
+
"mcp>=1.0.0,<2",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
macos = ["pyobjc-framework-Cocoa>=9.0", "pyobjc-framework-ApplicationServices>=9.0", "pyobjc-framework-Quartz>=9.0"]
|
|
35
|
+
windows = ["comtypes>=1.4"]
|
|
36
|
+
dev = ["pytest>=8.0", "pytest-asyncio>=0.23", "ruff>=0.9"]
|
|
37
|
+
|
|
38
|
+
[project.urls]
|
|
39
|
+
Homepage = "https://github.com/HoldMyBeer-gg/bad-ass-mcp"
|
|
40
|
+
Repository = "https://github.com/HoldMyBeer-gg/bad-ass-mcp"
|
|
41
|
+
Issues = "https://github.com/HoldMyBeer-gg/bad-ass-mcp/issues"
|
|
42
|
+
Changelog = "https://github.com/HoldMyBeer-gg/bad-ass-mcp/releases"
|
|
43
|
+
|
|
44
|
+
[project.scripts]
|
|
45
|
+
bad-ass-mcp = "bad_ass_mcp.server:main"
|
|
46
|
+
|
|
47
|
+
[tool.pytest.ini_options]
|
|
48
|
+
testpaths = ["tests"]
|
|
49
|
+
pythonpath = ["src"]
|
|
50
|
+
|
|
51
|
+
[tool.hatch.version]
|
|
52
|
+
source = "vcs"
|
|
53
|
+
|
|
54
|
+
[tool.ruff]
|
|
55
|
+
line-length = 100
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint]
|
|
58
|
+
select = ["E", "F", "I"]
|