winhands 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- winhands-0.3.0/LICENSE +21 -0
- winhands-0.3.0/PKG-INFO +194 -0
- winhands-0.3.0/README.md +159 -0
- winhands-0.3.0/pyproject.toml +52 -0
- winhands-0.3.0/setup.cfg +4 -0
- winhands-0.3.0/tests/test_cli.py +181 -0
- winhands-0.3.0/tests/test_inputs.py +74 -0
- winhands-0.3.0/tests/test_layout.py +23 -0
- winhands-0.3.0/tests/test_memory.py +41 -0
- winhands-0.3.0/tests/test_overlay.py +246 -0
- winhands-0.3.0/tests/test_overlay_anim.py +207 -0
- winhands-0.3.0/tests/test_server.py +77 -0
- winhands-0.3.0/tests/test_uia.py +317 -0
- winhands-0.3.0/tests/test_vision.py +95 -0
- winhands-0.3.0/winhands/SKILL.md +105 -0
- winhands-0.3.0/winhands/__init__.py +7 -0
- winhands-0.3.0/winhands/__main__.py +3 -0
- winhands-0.3.0/winhands/cli.py +103 -0
- winhands-0.3.0/winhands/inputs.py +311 -0
- winhands-0.3.0/winhands/memory.py +59 -0
- winhands-0.3.0/winhands/overlay.py +797 -0
- winhands-0.3.0/winhands/server.py +513 -0
- winhands-0.3.0/winhands/uia.py +789 -0
- winhands-0.3.0/winhands/vision.py +500 -0
- winhands-0.3.0/winhands.egg-info/PKG-INFO +194 -0
- winhands-0.3.0/winhands.egg-info/SOURCES.txt +28 -0
- winhands-0.3.0/winhands.egg-info/dependency_links.txt +1 -0
- winhands-0.3.0/winhands.egg-info/entry_points.txt +2 -0
- winhands-0.3.0/winhands.egg-info/requires.txt +17 -0
- winhands-0.3.0/winhands.egg-info/top_level.txt +1 -0
winhands-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rodolfo Obreque
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
winhands-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: winhands
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Computer use MCP server for Windows: a11y tree + pixels, code mode, game-grade input
|
|
5
|
+
Author: Rodolfo Obreque
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/zredo19/winhands
|
|
8
|
+
Project-URL: Repository, https://github.com/zredo19/winhands
|
|
9
|
+
Project-URL: Issues, https://github.com/zredo19/winhands/issues
|
|
10
|
+
Keywords: mcp,mcp-server,computer-use,windows,uiautomation,claude,claude-code,ai-agents
|
|
11
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: mcp>=2.2
|
|
19
|
+
Requires-Dist: uiautomation>=2.0.29
|
|
20
|
+
Requires-Dist: mss
|
|
21
|
+
Requires-Dist: Pillow
|
|
22
|
+
Requires-Dist: numpy
|
|
23
|
+
Requires-Dist: pywin32
|
|
24
|
+
Requires-Dist: opencv-python-headless
|
|
25
|
+
Requires-Dist: winrt-runtime
|
|
26
|
+
Requires-Dist: winrt-Windows.Media.Ocr
|
|
27
|
+
Requires-Dist: winrt-Windows.Graphics.Imaging
|
|
28
|
+
Requires-Dist: winrt-Windows.Storage.Streams
|
|
29
|
+
Requires-Dist: winrt-Windows.Foundation
|
|
30
|
+
Requires-Dist: winrt-Windows.Foundation.Collections
|
|
31
|
+
Requires-Dist: winrt-Windows.Globalization
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# winhands
|
|
37
|
+
|
|
38
|
+
[](https://github.com/zredo19/winhands/actions/workflows/ci.yml)
|
|
39
|
+
[](https://pypi.org/project/winhands/)
|
|
40
|
+
|
|
41
|
+
Computer use for Windows: a small MCP server that lets Claude (or any MCP client)
|
|
42
|
+
operate any desktop app, draw in canvases and drive games.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
Windows 10 2004+ or 11, and Claude Code or any MCP client. In PowerShell:
|
|
47
|
+
|
|
48
|
+
```powershell
|
|
49
|
+
winget install astral-sh.uv # once; installs uv, which brings its own Python (check the id with: winget search uv)
|
|
50
|
+
uv tool install winhands
|
|
51
|
+
winhands setup
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`winhands setup` registers the MCP server in Claude Code (user scope) and installs the skill into
|
|
55
|
+
`~/.claude/skills/winhands`. If `claude` is not on your PATH it prints the config snippet for other MCP clients instead.
|
|
56
|
+
Then restart Claude Code and ask it: "open Notepad and type hello".
|
|
57
|
+
|
|
58
|
+
Already have Python 3.11+? `pip install winhands`, then `winhands setup`.
|
|
59
|
+
|
|
60
|
+
- Update: `uv tool upgrade winhands`
|
|
61
|
+
- Uninstall: `winhands setup --remove`, then `uv tool uninstall winhands`
|
|
62
|
+
- See what `setup` would do without changing anything: `winhands setup --dry-run`
|
|
63
|
+
|
|
64
|
+
### Alternative: one-line installer
|
|
65
|
+
|
|
66
|
+
Installs uv if missing, then does the same as the commands above. You can read the
|
|
67
|
+
[script](install.ps1) first; it needs no admin rights.
|
|
68
|
+
|
|
69
|
+
```powershell
|
|
70
|
+
irm https://raw.githubusercontent.com/zredo19/winhands/main/install.ps1 | iex
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
To uninstall with it:
|
|
74
|
+
|
|
75
|
+
```powershell
|
|
76
|
+
& ([scriptblock]::Create((irm https://raw.githubusercontent.com/zredo19/winhands/main/install.ps1))) -Uninstall
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Prefer to do it by hand from the repo? See [Manual install](#manual-install).
|
|
80
|
+
|
|
81
|
+

|
|
82
|
+
*The overlay while Claude controls the PC: edge glow and status banner in four states (acting, thinking, paused, stopped).*
|
|
83
|
+
|
|
84
|
+

|
|
85
|
+
*The agent cursor at 2x: idle, moving (trail) and click (ripple), in the Claude, Antigravity and Codex palettes.*
|
|
86
|
+
|
|
87
|
+
Its design follows the computer-use harness of OpenAI's GPT-6 Astra in the Codex/ChatGPT desktop app,
|
|
88
|
+
and goes past it where that harness is weak (no key holds, no raw mouse, no local control loops):
|
|
89
|
+
|
|
90
|
+
1. **Accessibility tree first, pixels when needed.** A Codex-style UI Automation tree
|
|
91
|
+
(`12 button "Save" (focused)`), plus screenshots automatically when a window is canvas-like
|
|
92
|
+
(games, Paint), OCR, grid rulers, Set-of-Marks and burst montages.
|
|
93
|
+
2. **Code mode.** The model writes Python in a persistent REPL and batches many actions per turn,
|
|
94
|
+
including local perceive→act loops that run at 10–30 Hz without model round-trips.
|
|
95
|
+
3. **Built-in validation.** Every `run` returns a UI diff (or the full tree of a new dialog).
|
|
96
|
+
4. **Fail closed.** Element ids are bound to the latest snapshot; changed or vanished targets raise
|
|
97
|
+
`StaleTarget` instead of clicking the wrong thing.
|
|
98
|
+
5. **Takeover UX and safety.** An animated edge glow and status banner (acting, thinking, stopped; with the Ctrl+Alt+Q hint)
|
|
99
|
+
on the controlled monitor, plus an agent cursor with a motion trail and click ripple where real input lands, all in the
|
|
100
|
+
client's colour and excluded from screenshots.
|
|
101
|
+
Ctrl+LeftAlt+Q kill switch, and an automatic abort when the user touches mouse or keyboard.
|
|
102
|
+
Held inputs are released on abort, risky actions need confirmation, and some windows are denylisted.
|
|
103
|
+
6. **Memory.** Per-app notes and reusable skills (Voyager-style) that persist across sessions.
|
|
104
|
+
|
|
105
|
+
## Tools (2, ~815 tokens of schema)
|
|
106
|
+
|
|
107
|
+
| Tool | Modes / helpers |
|
|
108
|
+
|---|---|
|
|
109
|
+
| `observe(target, mode, region, grid, marks, frames, scale)` | `auto` (tree + shot if canvas-like), `tree`, `both`, `diff`, `shot`, `ocr`, `burst`, `windows` |
|
|
110
|
+
| `run(code, timeout, confirm)` | UIA: `click set_value action type key scroll find wait_for focus app windows sh` · Input: `press hold key_down key_up type_keys move move_rel click_at drag wheel release_all` · Vision: `show grab pixel find_color locate save_template ocr find_text click_text click_xy wait_change wait_stable` · Memory: `note notes save_skill skills` |
|
|
111
|
+
|
|
112
|
+
Input uses `SendInput` with scan codes (DirectInput/raw-input games see real keys) and raw relative
|
|
113
|
+
mouse for game cameras. Window capture uses `PrintWindow(PW_RENDERFULLCONTENT)`, so covered windows
|
|
114
|
+
are read correctly. Apps launched with `app()` survive the MCP session (launched through WMI,
|
|
115
|
+
outside the client's kill-on-close job).
|
|
116
|
+
|
|
117
|
+
## End-to-end results (real desktop, Windows 10, Spanish UI)
|
|
118
|
+
|
|
119
|
+
| Test | Result |
|
|
120
|
+
|---|---|
|
|
121
|
+
| Paint: draw a house (shapes by drag, palette swatches found by color, bucket fills), save PNG | 6/6 layout and color checks pass; one `run` of 11 s |
|
|
122
|
+
| Game probe (Chrome, pointer lock): scan codes W/A/S/D/Space/ShiftLeft | all six received as physical key codes |
|
|
123
|
+
| Raw relative mouse under pointer lock | sent (70, 30) → page received DX 70, DY 30 exactly |
|
|
124
|
+
| Buttons / wheel | L, M, R and wheel −3 received |
|
|
125
|
+
| 10 s aim test driven by one local loop (`find_color` → `click_at`) | 88 hits, 6 misses |
|
|
126
|
+
| OCR of known Notepad text | 30/34 words (88%) in 118 ms. Misses: the first glyph touching the edit border, and I/l confusion |
|
|
127
|
+
|
|
128
|
+
## Benchmark vs Windows-MCP
|
|
129
|
+
|
|
130
|
+
The same tasks run on the same machine, with each server driven the way an optimal agent would use it.
|
|
131
|
+
There were 3 reps per task. Script: [`bench/bench.py`](bench/bench.py), raw data:
|
|
132
|
+
[`bench/results.json`](bench/results.json).
|
|
133
|
+
|
|
134
|
+
| Task | Server | Success | Tool calls | Tool time | Tokens returned |
|
|
135
|
+
|---|---|---|---|---|---|
|
|
136
|
+
| Calculator 123×456 | **winhands** | **3/3** | **2** | **8.0 s** | **603** |
|
|
137
|
+
| | Windows-MCP 0.8.5 | 2/3 | 12 | 31.0 s | 6,060 |
|
|
138
|
+
| Notepad: type, Save As, verify file | **winhands** | **3/3** | **3** | **12.9 s** | **2,037** |
|
|
139
|
+
| | Windows-MCP 0.8.5 | 3/3 | 7 | 21.6 s | 8,601 |
|
|
140
|
+
|
|
141
|
+
The tool schema is sent on every turn: winhands uses about 815 tokens and Windows-MCP about 4,780.
|
|
142
|
+
|
|
143
|
+
Notes on method:
|
|
144
|
+
- Tokens are estimated: text is chars / 3.5, and images are `ceil(w/28)·ceil(h/28)` per Anthropic's vision docs.
|
|
145
|
+
- Tool time excludes model thinking. Each extra call also costs a model turn in a real agent.
|
|
146
|
+
- Windows-MCP's snapshot lists only interactive elements of the focused window, so reading the
|
|
147
|
+
calculator result needs a screenshot. Its failed rep missed the calculator in that snapshot.
|
|
148
|
+
- Windows-MCP dropped rapid consecutive clicks. A 1 s pause between its clicks, not counted, mimics agent pacing.
|
|
149
|
+
|
|
150
|
+
## Manual install
|
|
151
|
+
|
|
152
|
+
Windows 10 2004+ / 11, Python 3.11+. To run the development version from GitHub with [uv](https://docs.astral.sh/uv/):
|
|
153
|
+
|
|
154
|
+
```powershell
|
|
155
|
+
uv tool install --compile-bytecode git+https://github.com/zredo19/winhands
|
|
156
|
+
winhands setup
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
`winhands setup` registers the MCP server and installs the skill (perceive → batch → validate, game guidance, safety
|
|
160
|
+
rules). To do those two steps yourself: `claude mcp add winhands --scope user -- winhands`, and copy
|
|
161
|
+
[`winhands/SKILL.md`](winhands/SKILL.md) to `~/.claude/skills/winhands/SKILL.md`.
|
|
162
|
+
|
|
163
|
+
Any MCP client works: the server command is just `winhands` (stdio). Set `WINHANDS_OVERLAY=0` to hide the
|
|
164
|
+
overlay, `WINHANDS_LINGER=45` for how long it stays up between actions and `WINHANDS_SHARED=0` for strict mode. From source:
|
|
165
|
+
`pip install -e .` in a venv, then point the client at `python -m winhands`.
|
|
166
|
+
|
|
167
|
+
## Safety
|
|
168
|
+
|
|
169
|
+
- **Ctrl + Left Alt + Q** aborts the running action. AltGr+Q still types `@`.
|
|
170
|
+
- Shared mode (default): the user can keep working while a run acts through UIA patterns. Physical input
|
|
171
|
+
that collides with a run driving the real mouse/keyboard aborts it (`UserInterrupt`); real input waits for
|
|
172
|
+
the user to go idle first. The tool never fights the user for control. `WINHANDS_SHARED=0` restores strict mode.
|
|
173
|
+
- The low-level keyboard/mouse hooks behind both features exist only while a `run` is acting. Nothing hooks
|
|
174
|
+
system input while the server idles in a Claude Code session.
|
|
175
|
+
- Keys are only sent to a verified foreground target. All held keys and buttons are released on any abort.
|
|
176
|
+
- `sh()` and clicks on elements named like Send, Buy, Pay, Delete, Install or Allow require `run(..., confirm=True)`.
|
|
177
|
+
- Denylisted windows (password managers, banking) are refused.
|
|
178
|
+
- `run` executes arbitrary Python: the same privilege as Claude Code's shell tool, gated by the client's permission prompts.
|
|
179
|
+
- Online games with anti-cheat: automation may violate their terms and risk a ban. winhands makes
|
|
180
|
+
no attempt to hide or evade detection.
|
|
181
|
+
|
|
182
|
+
## Limits
|
|
183
|
+
|
|
184
|
+
- The harness makes a model faster and cheaper on the desktop. It does not make it smarter.
|
|
185
|
+
- Reflex gameplay needs local loops written by the model, or pausing the game. Model round-trips take seconds.
|
|
186
|
+
- Exclusive-fullscreen games may capture black; use borderless mode. Windows OCR skips isolated single
|
|
187
|
+
characters and glyphs touching strong borders. Template matching is exact-scale.
|
|
188
|
+
- Elevated (admin) windows ignore input from a non-elevated server (Windows UIPI).
|
|
189
|
+
|
|
190
|
+
## Tests
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
python -m pytest -q
|
|
194
|
+
```
|
winhands-0.3.0/README.md
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# winhands
|
|
2
|
+
|
|
3
|
+
[](https://github.com/zredo19/winhands/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/winhands/)
|
|
5
|
+
|
|
6
|
+
Computer use for Windows: a small MCP server that lets Claude (or any MCP client)
|
|
7
|
+
operate any desktop app, draw in canvases and drive games.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
Windows 10 2004+ or 11, and Claude Code or any MCP client. In PowerShell:
|
|
12
|
+
|
|
13
|
+
```powershell
|
|
14
|
+
winget install astral-sh.uv # once; installs uv, which brings its own Python (check the id with: winget search uv)
|
|
15
|
+
uv tool install winhands
|
|
16
|
+
winhands setup
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
`winhands setup` registers the MCP server in Claude Code (user scope) and installs the skill into
|
|
20
|
+
`~/.claude/skills/winhands`. If `claude` is not on your PATH it prints the config snippet for other MCP clients instead.
|
|
21
|
+
Then restart Claude Code and ask it: "open Notepad and type hello".
|
|
22
|
+
|
|
23
|
+
Already have Python 3.11+? `pip install winhands`, then `winhands setup`.
|
|
24
|
+
|
|
25
|
+
- Update: `uv tool upgrade winhands`
|
|
26
|
+
- Uninstall: `winhands setup --remove`, then `uv tool uninstall winhands`
|
|
27
|
+
- See what `setup` would do without changing anything: `winhands setup --dry-run`
|
|
28
|
+
|
|
29
|
+
### Alternative: one-line installer
|
|
30
|
+
|
|
31
|
+
Installs uv if missing, then does the same as the commands above. You can read the
|
|
32
|
+
[script](install.ps1) first; it needs no admin rights.
|
|
33
|
+
|
|
34
|
+
```powershell
|
|
35
|
+
irm https://raw.githubusercontent.com/zredo19/winhands/main/install.ps1 | iex
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
To uninstall with it:
|
|
39
|
+
|
|
40
|
+
```powershell
|
|
41
|
+
& ([scriptblock]::Create((irm https://raw.githubusercontent.com/zredo19/winhands/main/install.ps1))) -Uninstall
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Prefer to do it by hand from the repo? See [Manual install](#manual-install).
|
|
45
|
+
|
|
46
|
+

|
|
47
|
+
*The overlay while Claude controls the PC: edge glow and status banner in four states (acting, thinking, paused, stopped).*
|
|
48
|
+
|
|
49
|
+

|
|
50
|
+
*The agent cursor at 2x: idle, moving (trail) and click (ripple), in the Claude, Antigravity and Codex palettes.*
|
|
51
|
+
|
|
52
|
+
Its design follows the computer-use harness of OpenAI's GPT-6 Astra in the Codex/ChatGPT desktop app,
|
|
53
|
+
and goes past it where that harness is weak (no key holds, no raw mouse, no local control loops):
|
|
54
|
+
|
|
55
|
+
1. **Accessibility tree first, pixels when needed.** A Codex-style UI Automation tree
|
|
56
|
+
(`12 button "Save" (focused)`), plus screenshots automatically when a window is canvas-like
|
|
57
|
+
(games, Paint), OCR, grid rulers, Set-of-Marks and burst montages.
|
|
58
|
+
2. **Code mode.** The model writes Python in a persistent REPL and batches many actions per turn,
|
|
59
|
+
including local perceive→act loops that run at 10–30 Hz without model round-trips.
|
|
60
|
+
3. **Built-in validation.** Every `run` returns a UI diff (or the full tree of a new dialog).
|
|
61
|
+
4. **Fail closed.** Element ids are bound to the latest snapshot; changed or vanished targets raise
|
|
62
|
+
`StaleTarget` instead of clicking the wrong thing.
|
|
63
|
+
5. **Takeover UX and safety.** An animated edge glow and status banner (acting, thinking, stopped; with the Ctrl+Alt+Q hint)
|
|
64
|
+
on the controlled monitor, plus an agent cursor with a motion trail and click ripple where real input lands, all in the
|
|
65
|
+
client's colour and excluded from screenshots.
|
|
66
|
+
Ctrl+LeftAlt+Q kill switch, and an automatic abort when the user touches mouse or keyboard.
|
|
67
|
+
Held inputs are released on abort, risky actions need confirmation, and some windows are denylisted.
|
|
68
|
+
6. **Memory.** Per-app notes and reusable skills (Voyager-style) that persist across sessions.
|
|
69
|
+
|
|
70
|
+
## Tools (2, ~815 tokens of schema)
|
|
71
|
+
|
|
72
|
+
| Tool | Modes / helpers |
|
|
73
|
+
|---|---|
|
|
74
|
+
| `observe(target, mode, region, grid, marks, frames, scale)` | `auto` (tree + shot if canvas-like), `tree`, `both`, `diff`, `shot`, `ocr`, `burst`, `windows` |
|
|
75
|
+
| `run(code, timeout, confirm)` | UIA: `click set_value action type key scroll find wait_for focus app windows sh` · Input: `press hold key_down key_up type_keys move move_rel click_at drag wheel release_all` · Vision: `show grab pixel find_color locate save_template ocr find_text click_text click_xy wait_change wait_stable` · Memory: `note notes save_skill skills` |
|
|
76
|
+
|
|
77
|
+
Input uses `SendInput` with scan codes (DirectInput/raw-input games see real keys) and raw relative
|
|
78
|
+
mouse for game cameras. Window capture uses `PrintWindow(PW_RENDERFULLCONTENT)`, so covered windows
|
|
79
|
+
are read correctly. Apps launched with `app()` survive the MCP session (launched through WMI,
|
|
80
|
+
outside the client's kill-on-close job).
|
|
81
|
+
|
|
82
|
+
## End-to-end results (real desktop, Windows 10, Spanish UI)
|
|
83
|
+
|
|
84
|
+
| Test | Result |
|
|
85
|
+
|---|---|
|
|
86
|
+
| Paint: draw a house (shapes by drag, palette swatches found by color, bucket fills), save PNG | 6/6 layout and color checks pass; one `run` of 11 s |
|
|
87
|
+
| Game probe (Chrome, pointer lock): scan codes W/A/S/D/Space/ShiftLeft | all six received as physical key codes |
|
|
88
|
+
| Raw relative mouse under pointer lock | sent (70, 30) → page received DX 70, DY 30 exactly |
|
|
89
|
+
| Buttons / wheel | L, M, R and wheel −3 received |
|
|
90
|
+
| 10 s aim test driven by one local loop (`find_color` → `click_at`) | 88 hits, 6 misses |
|
|
91
|
+
| OCR of known Notepad text | 30/34 words (88%) in 118 ms. Misses: the first glyph touching the edit border, and I/l confusion |
|
|
92
|
+
|
|
93
|
+
## Benchmark vs Windows-MCP
|
|
94
|
+
|
|
95
|
+
The same tasks run on the same machine, with each server driven the way an optimal agent would use it.
|
|
96
|
+
There were 3 reps per task. Script: [`bench/bench.py`](bench/bench.py), raw data:
|
|
97
|
+
[`bench/results.json`](bench/results.json).
|
|
98
|
+
|
|
99
|
+
| Task | Server | Success | Tool calls | Tool time | Tokens returned |
|
|
100
|
+
|---|---|---|---|---|---|
|
|
101
|
+
| Calculator 123×456 | **winhands** | **3/3** | **2** | **8.0 s** | **603** |
|
|
102
|
+
| | Windows-MCP 0.8.5 | 2/3 | 12 | 31.0 s | 6,060 |
|
|
103
|
+
| Notepad: type, Save As, verify file | **winhands** | **3/3** | **3** | **12.9 s** | **2,037** |
|
|
104
|
+
| | Windows-MCP 0.8.5 | 3/3 | 7 | 21.6 s | 8,601 |
|
|
105
|
+
|
|
106
|
+
The tool schema is sent on every turn: winhands uses about 815 tokens and Windows-MCP about 4,780.
|
|
107
|
+
|
|
108
|
+
Notes on method:
|
|
109
|
+
- Tokens are estimated: text is chars / 3.5, and images are `ceil(w/28)·ceil(h/28)` per Anthropic's vision docs.
|
|
110
|
+
- Tool time excludes model thinking. Each extra call also costs a model turn in a real agent.
|
|
111
|
+
- Windows-MCP's snapshot lists only interactive elements of the focused window, so reading the
|
|
112
|
+
calculator result needs a screenshot. Its failed rep missed the calculator in that snapshot.
|
|
113
|
+
- Windows-MCP dropped rapid consecutive clicks. A 1 s pause between its clicks, not counted, mimics agent pacing.
|
|
114
|
+
|
|
115
|
+
## Manual install
|
|
116
|
+
|
|
117
|
+
Windows 10 2004+ / 11, Python 3.11+. To run the development version from GitHub with [uv](https://docs.astral.sh/uv/):
|
|
118
|
+
|
|
119
|
+
```powershell
|
|
120
|
+
uv tool install --compile-bytecode git+https://github.com/zredo19/winhands
|
|
121
|
+
winhands setup
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`winhands setup` registers the MCP server and installs the skill (perceive → batch → validate, game guidance, safety
|
|
125
|
+
rules). To do those two steps yourself: `claude mcp add winhands --scope user -- winhands`, and copy
|
|
126
|
+
[`winhands/SKILL.md`](winhands/SKILL.md) to `~/.claude/skills/winhands/SKILL.md`.
|
|
127
|
+
|
|
128
|
+
Any MCP client works: the server command is just `winhands` (stdio). Set `WINHANDS_OVERLAY=0` to hide the
|
|
129
|
+
overlay, `WINHANDS_LINGER=45` for how long it stays up between actions and `WINHANDS_SHARED=0` for strict mode. From source:
|
|
130
|
+
`pip install -e .` in a venv, then point the client at `python -m winhands`.
|
|
131
|
+
|
|
132
|
+
## Safety
|
|
133
|
+
|
|
134
|
+
- **Ctrl + Left Alt + Q** aborts the running action. AltGr+Q still types `@`.
|
|
135
|
+
- Shared mode (default): the user can keep working while a run acts through UIA patterns. Physical input
|
|
136
|
+
that collides with a run driving the real mouse/keyboard aborts it (`UserInterrupt`); real input waits for
|
|
137
|
+
the user to go idle first. The tool never fights the user for control. `WINHANDS_SHARED=0` restores strict mode.
|
|
138
|
+
- The low-level keyboard/mouse hooks behind both features exist only while a `run` is acting. Nothing hooks
|
|
139
|
+
system input while the server idles in a Claude Code session.
|
|
140
|
+
- Keys are only sent to a verified foreground target. All held keys and buttons are released on any abort.
|
|
141
|
+
- `sh()` and clicks on elements named like Send, Buy, Pay, Delete, Install or Allow require `run(..., confirm=True)`.
|
|
142
|
+
- Denylisted windows (password managers, banking) are refused.
|
|
143
|
+
- `run` executes arbitrary Python: the same privilege as Claude Code's shell tool, gated by the client's permission prompts.
|
|
144
|
+
- Online games with anti-cheat: automation may violate their terms and risk a ban. winhands makes
|
|
145
|
+
no attempt to hide or evade detection.
|
|
146
|
+
|
|
147
|
+
## Limits
|
|
148
|
+
|
|
149
|
+
- The harness makes a model faster and cheaper on the desktop. It does not make it smarter.
|
|
150
|
+
- Reflex gameplay needs local loops written by the model, or pausing the game. Model round-trips take seconds.
|
|
151
|
+
- Exclusive-fullscreen games may capture black; use borderless mode. Windows OCR skips isolated single
|
|
152
|
+
characters and glyphs touching strong borders. Template matching is exact-scale.
|
|
153
|
+
- Elevated (admin) windows ignore input from a non-elevated server (Windows UIPI).
|
|
154
|
+
|
|
155
|
+
## Tests
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
python -m pytest -q
|
|
159
|
+
```
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"] # PEP 639: license = "MIT" as an SPDX string
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "winhands"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Computer use MCP server for Windows: a11y tree + pixels, code mode, game-grade input"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "Rodolfo Obreque" }]
|
|
13
|
+
keywords = ["mcp", "mcp-server", "computer-use", "windows", "uiautomation", "claude", "claude-code", "ai-agents"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Operating System :: Microsoft :: Windows",
|
|
16
|
+
"Programming Language :: Python :: 3.11",
|
|
17
|
+
"Programming Language :: Python :: 3.12",
|
|
18
|
+
"Topic :: Software Development :: Libraries",
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"mcp>=2.2",
|
|
22
|
+
"uiautomation>=2.0.29",
|
|
23
|
+
"mss",
|
|
24
|
+
"Pillow",
|
|
25
|
+
"numpy",
|
|
26
|
+
"pywin32",
|
|
27
|
+
"opencv-python-headless",
|
|
28
|
+
"winrt-runtime",
|
|
29
|
+
"winrt-Windows.Media.Ocr",
|
|
30
|
+
"winrt-Windows.Graphics.Imaging",
|
|
31
|
+
"winrt-Windows.Storage.Streams",
|
|
32
|
+
"winrt-Windows.Foundation",
|
|
33
|
+
"winrt-Windows.Foundation.Collections",
|
|
34
|
+
"winrt-Windows.Globalization",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/zredo19/winhands"
|
|
39
|
+
Repository = "https://github.com/zredo19/winhands"
|
|
40
|
+
Issues = "https://github.com/zredo19/winhands/issues"
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
winhands = "winhands.cli:main"
|
|
44
|
+
|
|
45
|
+
[project.optional-dependencies]
|
|
46
|
+
dev = ["pytest"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools]
|
|
49
|
+
packages = ["winhands"]
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.package-data]
|
|
52
|
+
winhands = ["SKILL.md"]
|
winhands-0.3.0/setup.cfg
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import sys, pathlib, json, importlib
|
|
2
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from winhands import cli
|
|
7
|
+
|
|
8
|
+
EXE = r"C:\x\Scripts\winhands.exe"
|
|
9
|
+
CLAUDE = r"C:\bin\claude.cmd"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Run:
|
|
13
|
+
"""Fake subprocess.run: records argv, answers returncode per call index (default 0)."""
|
|
14
|
+
def __init__(self, *codes):
|
|
15
|
+
self.calls, self.codes = [], list(codes)
|
|
16
|
+
|
|
17
|
+
def __call__(self, cmd, **kw):
|
|
18
|
+
self.calls.append(list(cmd))
|
|
19
|
+
code = self.codes[len(self.calls) - 1] if len(self.calls) <= len(self.codes) else 0
|
|
20
|
+
return type("R", (), {"returncode": code})()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _which(**found):
|
|
24
|
+
return lambda name: found.get(name)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@pytest.fixture(autouse=True)
|
|
28
|
+
def no_real_warm_up(monkeypatch):
|
|
29
|
+
monkeypatch.setattr(cli, "_warm", lambda: None) # the real one imports the whole server (seconds)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_setup_warms_up_the_install_so_the_first_mcp_start_is_fast(tmp_path):
|
|
33
|
+
calls = []
|
|
34
|
+
cli.setup(home=tmp_path, which=_which(claude=CLAUDE), run=Run(), command=[EXE], out=lambda s: None,
|
|
35
|
+
warm=lambda: calls.append(1))
|
|
36
|
+
assert calls == [1]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_warm_up_is_skipped_for_dry_run_and_remove(tmp_path):
|
|
40
|
+
def boom():
|
|
41
|
+
pytest.fail("warmed up")
|
|
42
|
+
for kw in ({"dry_run": True}, {"remove": True}):
|
|
43
|
+
assert cli.setup(home=tmp_path, which=_which(), run=Run(), command=[EXE], out=lambda s: None, warm=boom, **kw) == 0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_a_failing_warm_up_never_fails_setup(tmp_path):
|
|
47
|
+
def boom():
|
|
48
|
+
raise RuntimeError("no display")
|
|
49
|
+
out = []
|
|
50
|
+
assert cli.setup(home=tmp_path, which=_which(claude=CLAUDE), run=Run(), command=[EXE], out=out.append, warm=boom) == 0
|
|
51
|
+
assert any("Warm-up skipped" in line and "no display" in line for line in out)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_no_arguments_starts_the_server_and_never_builds_a_parser(monkeypatch):
|
|
55
|
+
import argparse
|
|
56
|
+
monkeypatch.setattr(argparse, "ArgumentParser", lambda *a, **k: pytest.fail("argparse touched"))
|
|
57
|
+
served = []
|
|
58
|
+
assert cli.main([], serve=lambda: served.append(1)) == 0
|
|
59
|
+
assert served == [1]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_version_prints_the_package_version(capsys):
|
|
63
|
+
import winhands
|
|
64
|
+
assert cli.main(["--version"]) == 0
|
|
65
|
+
assert capsys.readouterr().out.strip() == f"winhands {winhands.__version__}"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_package_version_falls_back_when_not_installed(monkeypatch):
|
|
69
|
+
import importlib.metadata as md
|
|
70
|
+
import winhands
|
|
71
|
+
|
|
72
|
+
def missing(name):
|
|
73
|
+
raise md.PackageNotFoundError(name)
|
|
74
|
+
monkeypatch.setattr(md, "version", missing)
|
|
75
|
+
importlib.reload(winhands)
|
|
76
|
+
try:
|
|
77
|
+
assert winhands.__version__ == "unknown"
|
|
78
|
+
finally:
|
|
79
|
+
monkeypatch.undo()
|
|
80
|
+
importlib.reload(winhands)
|
|
81
|
+
assert winhands.__version__ != "unknown"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_skill_text_is_the_packaged_skill():
|
|
85
|
+
text = cli.skill_text()
|
|
86
|
+
assert text.startswith("---") and "name: winhands" in text
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_setup_registers_mcp_and_installs_skill(tmp_path):
|
|
90
|
+
run, out = Run(), []
|
|
91
|
+
assert cli.setup(home=tmp_path, which=_which(claude=CLAUDE), run=run, command=[EXE], out=out.append) == 0
|
|
92
|
+
assert run.calls == [[CLAUDE, "mcp", "remove", "winhands", "--scope", "user"],
|
|
93
|
+
[CLAUDE, "mcp", "add", "winhands", "--scope", "user", "--", EXE]]
|
|
94
|
+
skill = tmp_path / ".claude" / "skills" / "winhands" / "SKILL.md"
|
|
95
|
+
assert skill.read_text(encoding="utf-8") == cli.skill_text()
|
|
96
|
+
assert any("Restart Claude Code" in line for line in out)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_setup_is_idempotent_when_remove_fails(tmp_path):
|
|
100
|
+
run = Run(1, 0) # the entry did not exist yet: remove fails, add works
|
|
101
|
+
assert cli.setup(home=tmp_path, which=_which(claude=CLAUDE), run=run, command=[EXE], out=lambda s: None) == 0
|
|
102
|
+
assert len(run.calls) == 2
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_setup_reports_a_failed_add(tmp_path):
|
|
106
|
+
out = []
|
|
107
|
+
run = Run(0, 3)
|
|
108
|
+
assert cli.setup(home=tmp_path, which=_which(claude=CLAUDE), run=run, command=[EXE], out=out.append) == 1
|
|
109
|
+
assert any("claude mcp add" in line for line in out)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_setup_without_claude_prints_the_json_snippet(tmp_path):
|
|
113
|
+
out = []
|
|
114
|
+
run = Run()
|
|
115
|
+
assert cli.setup(home=tmp_path, which=_which(), run=run, command=[EXE], out=out.append) == 0
|
|
116
|
+
assert run.calls == []
|
|
117
|
+
blob = next(line for line in out if line.lstrip().startswith("{"))
|
|
118
|
+
assert json.loads(blob) == {"mcpServers": {"winhands": {"command": EXE}}}
|
|
119
|
+
assert (tmp_path / ".claude" / "skills" / "winhands" / "SKILL.md").is_file()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_snippet_carries_args_for_the_python_dash_m_fallback(tmp_path):
|
|
123
|
+
out = []
|
|
124
|
+
cli.setup(home=tmp_path, which=_which(), run=Run(), command=[r"C:\py\python.exe", "-m", "winhands"], out=out.append)
|
|
125
|
+
blob = next(line for line in out if line.lstrip().startswith("{"))
|
|
126
|
+
assert json.loads(blob)["mcpServers"]["winhands"] == {"command": r"C:\py\python.exe", "args": ["-m", "winhands"]}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_setup_dry_run_changes_nothing(tmp_path):
|
|
130
|
+
out = []
|
|
131
|
+
assert cli.setup(dry_run=True, home=tmp_path, which=_which(claude=CLAUDE),
|
|
132
|
+
run=lambda *a, **k: pytest.fail("ran a command"), command=[EXE], out=out.append) == 0
|
|
133
|
+
assert list(tmp_path.iterdir()) == []
|
|
134
|
+
assert any("dry run" in line.lower() for line in out)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_remove_unregisters_and_deletes_the_skill(tmp_path):
|
|
138
|
+
d = tmp_path / ".claude" / "skills" / "winhands"
|
|
139
|
+
d.mkdir(parents=True)
|
|
140
|
+
(d / "SKILL.md").write_text("x", encoding="utf-8")
|
|
141
|
+
run = Run()
|
|
142
|
+
assert cli.setup(remove=True, home=tmp_path, which=_which(claude=CLAUDE), run=run, command=[EXE],
|
|
143
|
+
out=lambda s: None) == 0
|
|
144
|
+
assert run.calls == [[CLAUDE, "mcp", "remove", "winhands", "--scope", "user"]]
|
|
145
|
+
assert not d.exists()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def test_remove_is_fine_with_nothing_installed_and_no_claude(tmp_path):
|
|
149
|
+
assert cli.setup(remove=True, home=tmp_path, which=_which(), run=Run(), command=[EXE], out=lambda s: None) == 0
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_main_passes_setup_flags_through(monkeypatch):
|
|
153
|
+
seen = {}
|
|
154
|
+
monkeypatch.setattr(cli, "setup", lambda **kw: seen.update(kw) or 7)
|
|
155
|
+
assert cli.main(["setup", "--remove", "--dry-run"]) == 7
|
|
156
|
+
assert seen == {"remove": True, "dry_run": True}
|
|
157
|
+
assert cli.main(["setup"]) == 7
|
|
158
|
+
assert seen == {"remove": False, "dry_run": False}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_unknown_command_is_an_argparse_error():
|
|
162
|
+
with pytest.raises(SystemExit) as e:
|
|
163
|
+
cli.main(["bogus"])
|
|
164
|
+
assert e.value.code == 2
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_command_is_the_running_executable_when_it_is_winhands():
|
|
168
|
+
got = cli.command_for(argv0=EXE, which=_which(winhands=r"C:\other\winhands.exe"),
|
|
169
|
+
executable=r"C:\py\python.exe", isfile=lambda p: True)
|
|
170
|
+
assert got == [EXE]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def test_command_falls_back_to_path_then_scripts_dir_then_python_dash_m():
|
|
174
|
+
py = r"C:\py\python.exe"
|
|
175
|
+
other = r"C:\other\winhands.exe"
|
|
176
|
+
# `python -m winhands`: argv0 is the package's __main__.py, not the executable
|
|
177
|
+
assert cli.command_for(argv0=r"C:\p\winhands\__main__.py", which=_which(winhands=other),
|
|
178
|
+
executable=py, isfile=lambda p: True) == [other]
|
|
179
|
+
scripts = str(pathlib.Path(py).parent / "winhands.exe")
|
|
180
|
+
assert cli.command_for(argv0="x", which=_which(), executable=py, isfile=lambda p: p == scripts) == [scripts]
|
|
181
|
+
assert cli.command_for(argv0="x", which=_which(), executable=py, isfile=lambda p: False) == [py, "-m", "winhands"]
|