doppelhand 1.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doppelhand-1.3.1/CHANGELOG.md +61 -0
- doppelhand-1.3.1/LICENSE +21 -0
- doppelhand-1.3.1/MANIFEST.in +3 -0
- doppelhand-1.3.1/PKG-INFO +191 -0
- doppelhand-1.3.1/README.md +160 -0
- doppelhand-1.3.1/bench/benchmark.py +81 -0
- doppelhand-1.3.1/pyproject.toml +49 -0
- doppelhand-1.3.1/setup.cfg +4 -0
- doppelhand-1.3.1/src/doppelhand/__init__.py +20 -0
- doppelhand-1.3.1/src/doppelhand/agent.py +174 -0
- doppelhand-1.3.1/src/doppelhand/cli.py +461 -0
- doppelhand-1.3.1/src/doppelhand/duplication.py +387 -0
- doppelhand-1.3.1/src/doppelhand/errors.py +31 -0
- doppelhand-1.3.1/src/doppelhand/executor.py +299 -0
- doppelhand-1.3.1/src/doppelhand/inputs.py +350 -0
- doppelhand-1.3.1/src/doppelhand/screen.py +381 -0
- doppelhand-1.3.1/src/doppelhand/serve.py +215 -0
- doppelhand-1.3.1/src/doppelhand/session.py +62 -0
- doppelhand-1.3.1/src/doppelhand/skill/SKILL.md +123 -0
- doppelhand-1.3.1/src/doppelhand/skills.py +47 -0
- doppelhand-1.3.1/src/doppelhand.egg-info/PKG-INFO +191 -0
- doppelhand-1.3.1/src/doppelhand.egg-info/SOURCES.txt +32 -0
- doppelhand-1.3.1/src/doppelhand.egg-info/dependency_links.txt +1 -0
- doppelhand-1.3.1/src/doppelhand.egg-info/entry_points.txt +2 -0
- doppelhand-1.3.1/src/doppelhand.egg-info/requires.txt +8 -0
- doppelhand-1.3.1/src/doppelhand.egg-info/top_level.txt +1 -0
- doppelhand-1.3.1/tests/conftest.py +4 -0
- doppelhand-1.3.1/tests/fakes.py +131 -0
- doppelhand-1.3.1/tests/test_agent.py +135 -0
- doppelhand-1.3.1/tests/test_cli.py +261 -0
- doppelhand-1.3.1/tests/test_executor.py +192 -0
- doppelhand-1.3.1/tests/test_frames.py +111 -0
- doppelhand-1.3.1/tests/test_keys.py +83 -0
- doppelhand-1.3.1/tests/test_serve.py +172 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 1.3.1 — 2026-09-09
|
|
4
|
+
|
|
5
|
+
First release on PyPI: `pip install doppelhand`.
|
|
6
|
+
|
|
7
|
+
- The Anthropic SDK is no longer installed by default. Only `doppelhand run` talks to an API, so it moved to an extra: `pip install "doppelhand[run]"`. Running it without the SDK now says so instead of raising an import error. The floor is `anthropic>=0.124`, the first version carrying the computer-toolset types.
|
|
8
|
+
- A rotated display falls back to GDI capture. Desktop Duplication hands over the frame in the screen's native orientation, so a portrait monitor would have produced a sideways image whose coordinates disagreed with the desktop rectangle.
|
|
9
|
+
- The source archive now carries the whole test suite, the benchmark and the changelog, so a clean checkout can run what the repository runs.
|
|
10
|
+
- The warning about driving the real mouse and keyboard sits under Install, where someone arriving from the package page will see it.
|
|
11
|
+
|
|
12
|
+
## 1.3.0 — 2026-09-09
|
|
13
|
+
|
|
14
|
+
Built for driving a desktop in real time. An action costs about 6ms instead of 300, and a screenshot about 30ms instead of 400.
|
|
15
|
+
|
|
16
|
+
Measured first: a command that did no work at all cost ~400ms, of which ~300ms was Python starting and importing, while capturing the screen through GDI cost 150-775ms and varied wildly from call to call. Resizing and encoding, the parts that looked expensive, were never more than 50ms together.
|
|
17
|
+
|
|
18
|
+
- `doppelhand serve` holds one warm process open and answers the same actions over loopback HTTP. Reads are GET, anything that moves the pointer or types is POST, and the arguments keep the names the command line uses. The query string is turned back into a command line and handed to the same parser and the same functions, so the two surfaces cannot drift apart.
|
|
19
|
+
- Screens are captured through DXGI Desktop Duplication, which hands over the frame the compositor already holds instead of asking the system to read the screen back. Held open by the server, a capture costs microseconds rather than hundreds of milliseconds, and an unchanged screen costs nothing at all. Opening a duplication is expensive, so one-shot commands keep using GDI, and any display that refuses duplication falls back to it too.
|
|
20
|
+
- The pointer is composited onto duplicated frames, which arrive without one.
|
|
21
|
+
- `shot --fast` writes JPEG and resizes by box filter, roughly a third of the time for slightly softer text.
|
|
22
|
+
- A shot reports which path produced it in `source`.
|
|
23
|
+
- The argument parser is built once in a long-lived process, which was costing 13ms of every request.
|
|
24
|
+
- Usage errors return an exit code instead of raising, so a bad argument reads the same as any other failure.
|
|
25
|
+
|
|
26
|
+
The server binds to loopback only, requires a secret it writes to a file in the user's own profile, refuses any request carrying browser headers, and refuses any request whose Host is not loopback, which is what stops a web page reaching it by name.
|
|
27
|
+
|
|
28
|
+
## 1.2.0 — 2026-09-09
|
|
29
|
+
|
|
30
|
+
Every display is reachable, the pointer is visible, and a coordinate that misses now says so.
|
|
31
|
+
|
|
32
|
+
- A point outside the view is refused instead of being clamped to the nearest edge. Clamping meant a scale mistake clicked a screen corner and still reported success, which on Windows 11 is the show-desktop hotspot.
|
|
33
|
+
- Usage errors print the same JSON shape as every other failure, so a caller parsing stdout never has to fall back to reading argparse's prose. They exit 2, while a refused action exits 1.
|
|
34
|
+
- `screen` lists every display, numbered left to right. `--monitor N` picks one and `--monitor all` captures them as a single wide image. Each display has its own view space starting at 0,0, so coordinates stay positive whatever the desktop layout.
|
|
35
|
+
- Screenshots include the mouse pointer, composited in after the capture because a GDI copy never contains it. `shot --no-cursor` leaves it out.
|
|
36
|
+
- `cursor` reports which display the pointer is actually on, not just where it is in the current view.
|
|
37
|
+
- The display of the last shot is remembered along with its size. Moving or unplugging a monitor discards the memory instead of misplacing a click.
|
|
38
|
+
- Input refused by a background desktop is retried after attaching to the input desktop, sending only the events that did not get through.
|
|
39
|
+
|
|
40
|
+
## 1.1.0 — 2026-09-09
|
|
41
|
+
|
|
42
|
+
Any AI harness can now use doppelhand as its hands, with no API key and no second model.
|
|
43
|
+
|
|
44
|
+
- Each action is its own command: `shot`, `screen`, `click`, `move`, `drag`, `scroll`, `type`, `key`, `hold`, `wait` and `cursor`. They print one JSON object and exit non-zero when the action did not happen.
|
|
45
|
+
- `shot` writes a PNG and reports the coordinate space, so the calling agent reads the image with its own tools and answers in the pixels it saw.
|
|
46
|
+
- Coordinates default to the screenshot's space and are scaled back to the display. `--space display` passes real pixels straight through.
|
|
47
|
+
- The size of the last full screenshot is remembered, so a click after a `--max-edge` shot lands in the space that shot was taken in.
|
|
48
|
+
- `install-skill` drops a portable agent skill into Claude Code, opencode, AGY (global or workspace), or any directory given with `--dest`. `--print` writes it to stdout.
|
|
49
|
+
- `run` is unchanged and remains the only path that calls the Anthropic API.
|
|
50
|
+
|
|
51
|
+
## 1.0.0 — 2026-09-09
|
|
52
|
+
|
|
53
|
+
First release.
|
|
54
|
+
|
|
55
|
+
- `doppelhand run "<task>"` drives the desktop from a plain-language instruction, using Claude's computer toolset.
|
|
56
|
+
- `doppelhand shot` saves a screenshot without calling the API.
|
|
57
|
+
- Screen capture through GDI and input through `SendInput`, with no capture or automation dependency.
|
|
58
|
+
- All seventeen toolset actions are carried out: screenshot, zoom, the five click kinds, drag, move, button down and up, cursor position, scroll, type, key, hold key and wait.
|
|
59
|
+
- Coordinates are scaled between screenshot pixels and display pixels, and the process declares per-monitor DPI awareness so the two agree on scaled displays.
|
|
60
|
+
- A run stops when Escape is held, when the step budget runs out, or when the model declines the task.
|
|
61
|
+
- Screenshot history is pruned in batches and a prompt-cache breakpoint follows the newest tool result.
|
doppelhand-1.3.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shanewas Ahmed
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: doppelhand
|
|
3
|
+
Version: 1.3.1
|
|
4
|
+
Summary: Windows agent that reads the screen and drives the mouse and keyboard like a person would
|
|
5
|
+
Author-email: Shanewas Ahmed <shanewasahmed@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/shanewas/doppelhand
|
|
8
|
+
Project-URL: Issues, https://github.com/shanewas/doppelhand/issues
|
|
9
|
+
Keywords: windows,automation,computer-use,screenshot,agent
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Environment :: Win32 (MS Windows)
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Desktop Environment
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pillow>=10
|
|
25
|
+
Provides-Extra: run
|
|
26
|
+
Requires-Dist: anthropic>=0.124; extra == "run"
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
29
|
+
Requires-Dist: anthropic>=0.124; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# doppelhand
|
|
33
|
+
|
|
34
|
+
A double of you at the keyboard. doppelhand captures the Windows screen and drives the mouse and keyboard: clicking, dragging, scrolling and typing the way a person would.
|
|
35
|
+
|
|
36
|
+
Use it two ways. Give an AI agent you already have eyes and hands, so it looks at your desktop with its own model and acts through doppelhand. Or let doppelhand drive itself with Claude, using its own key.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
pip install doppelhand
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Windows only, Python 3.10 or newer. Pillow is the only dependency; screen capture and
|
|
45
|
+
input go straight through the Win32 API.
|
|
46
|
+
|
|
47
|
+
**This program moves your real mouse and types on your real keyboard.** It can click
|
|
48
|
+
anything you can click. Run it on a machine where that is acceptable and watch what it
|
|
49
|
+
does. `doppelhand run` waits for confirmation and stops when you hold Escape; the
|
|
50
|
+
single-action commands do what they are told, immediately.
|
|
51
|
+
|
|
52
|
+
## Give your agent hands
|
|
53
|
+
|
|
54
|
+
Install the skill into whichever harness you use, and it learns the loop by itself:
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
doppelhand install-skill claude-code
|
|
58
|
+
doppelhand install-skill opencode
|
|
59
|
+
doppelhand install-skill agy # global, ~/.gemini/config/skills
|
|
60
|
+
doppelhand install-skill agy-workspace # this project's .agents/skills
|
|
61
|
+
doppelhand install-skill --dest path/to/skills/doppelhand # anything else
|
|
62
|
+
doppelhand install-skill --print # read it first
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
No API key is involved. The agent takes a screenshot, reads the PNG with its own image tool, and calls back with coordinates. Every command answers with one JSON object and exits non-zero when the action did not happen.
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
$ doppelhand shot
|
|
69
|
+
{"ok": true, "path": "C:\\Users\\you\\AppData\\Local\\doppelhand\\shot.png",
|
|
70
|
+
"view": [1280, 720], "display": [1920, 1080], "scale": 0.666667, "space": "view"}
|
|
71
|
+
|
|
72
|
+
$ doppelhand click 640,360
|
|
73
|
+
{"ok": true, "action": "left_click", "at": [640, 360]}
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
| Command | What it does |
|
|
77
|
+
|---|---|
|
|
78
|
+
| `shot [PATH] [--region X,Y,W,H] [--monitor N]` | capture, write a PNG, report the coordinate space |
|
|
79
|
+
| `screen` | list the displays and report the coordinate space |
|
|
80
|
+
| `click X,Y [--button] [--count] [--modifiers]` | click, double click, right click |
|
|
81
|
+
| `move X,Y` / `drag X1,Y1 X2,Y2` | move the pointer, or press and drag |
|
|
82
|
+
| `scroll up\|down\|left\|right [N] [--at X,Y]` | scroll |
|
|
83
|
+
| `type "text"` / `key "ctrl+s" [--repeat N]` | type text, press a combination |
|
|
84
|
+
| `hold "shift" 2` / `wait 1.5` | hold a key, or pause |
|
|
85
|
+
| `cursor` | where the pointer is, in both spaces |
|
|
86
|
+
|
|
87
|
+
Coordinates are the pixels of the screenshot, not of the display; doppelhand scales them back up. A point outside that view is refused rather than clamped, so a scale mistake fails loudly instead of clicking a corner. Pass `--space display` if you already have real display pixels.
|
|
88
|
+
|
|
89
|
+
Every display is addressable. `screen` lists them numbered left to right, `--monitor N` picks one, and `--monitor all` captures the lot as a single wide image. Each display has its own view space starting at 0,0, so coordinates never go negative. The display and size of the last shot are remembered, so the next command lands in the same space without repeating flags; rearranging or unplugging a monitor discards that memory rather than misplacing a click.
|
|
90
|
+
|
|
91
|
+
Screenshots include the mouse pointer, which a GDI capture leaves out by default. Turn it off with `shot --no-cursor`.
|
|
92
|
+
|
|
93
|
+
## Real time
|
|
94
|
+
|
|
95
|
+
A fresh process costs about 300ms before it does anything, which dominates a session of
|
|
96
|
+
many small actions. Run a server once and talk to it over loopback instead:
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
doppelhand serve
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
It prints a port and a secret, and writes both to `%LOCALAPPDATA%\doppelhand\serve.json`.
|
|
103
|
+
Reads are GET, anything that moves the pointer or types is POST, and arguments keep the
|
|
104
|
+
names the command line uses:
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
curl -sH "X-Doppelhand-Token: $TOKEN" "http://127.0.0.1:$PORT/shot?fast=1"
|
|
108
|
+
curl -sXPOST -H "X-Doppelhand-Token: $TOKEN" "http://127.0.0.1:$PORT/click?at=640,360"
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
One run of `python bench/benchmark.py` on a three-monitor desktop. The spawned column
|
|
112
|
+
moves a lot with what the machine is doing, so re-measure rather than trusting these:
|
|
113
|
+
|
|
114
|
+
| action | spawned | served | |
|
|
115
|
+
|---|---|---|---|
|
|
116
|
+
| `cursor` | 309 ms | 5.6 ms | 55× |
|
|
117
|
+
| `screen` | 262 ms | 5.3 ms | 49× |
|
|
118
|
+
| `move` | 262 ms | 5.9 ms | 44× |
|
|
119
|
+
| `shot` | 430 ms | 77 ms | 5.5× |
|
|
120
|
+
| `shot --fast` | 378 ms | 30 ms | 12× |
|
|
121
|
+
|
|
122
|
+
Most of the screenshot gain comes from capturing through DXGI Desktop Duplication, which
|
|
123
|
+
hands over the frame the compositor already holds. Opening one costs more than a whole
|
|
124
|
+
GDI capture, so it only pays inside the server; one-shot commands keep the GDI path, and
|
|
125
|
+
so does any display that refuses to be duplicated. Every shot reports which it used in
|
|
126
|
+
`source`.
|
|
127
|
+
|
|
128
|
+
The server binds to loopback only, needs the secret from that file, and turns away any
|
|
129
|
+
request that carries browser headers or names a host other than loopback.
|
|
130
|
+
|
|
131
|
+
## Or let it drive itself
|
|
132
|
+
|
|
133
|
+
This is the only path that calls the Anthropic API, the only one that needs a key, and
|
|
134
|
+
the only one that needs the SDK:
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
pip install "doppelhand[run]"
|
|
138
|
+
export ANTHROPIC_API_KEY=sk-ant-...
|
|
139
|
+
doppelhand run "open the calculator and work out 19% of 4,320"
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
This loop is covered by tests against a scripted client rather than a recorded live run,
|
|
143
|
+
so treat it as the least proven part of the package.
|
|
144
|
+
|
|
145
|
+
`run` prints what it is about to do and waits for confirmation. **Hold Escape at any point to stop the run** — the key is checked before every action.
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
--max-steps N model turns before giving up (default 30)
|
|
149
|
+
--max-edge N long edge of the screenshots sent to the model (default 1280)
|
|
150
|
+
--model NAME defaults to claude-opus-5
|
|
151
|
+
-y skip the confirmation
|
|
152
|
+
-q print only the final answer
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
As a library:
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
from doppelhand.agent import Agent
|
|
159
|
+
|
|
160
|
+
print(Agent(max_steps=10).run("close the notification in the corner"))
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## How it works
|
|
164
|
+
|
|
165
|
+
1. `screen.py` captures a display, through GDI for a one-shot command or through DXGI Desktop Duplication when the server is holding one open, and hands back a Pillow image.
|
|
166
|
+
2. `executor.py` shrinks that image to a 1280-pixel long edge, and scales every coordinate the model returns back up to real pixels.
|
|
167
|
+
3. `inputs.py` drives the mouse and keyboard with `SendInput`. Text is typed as Unicode, so it does not depend on the active keyboard layout.
|
|
168
|
+
4. `cli.py` exposes each of those actions as a command, or `agent.py` runs the whole loop against Claude: send the screen, execute the actions in the reply, send the result, repeat.
|
|
169
|
+
|
|
170
|
+
In the built-in loop, costs are kept down two ways: one prompt-cache breakpoint moves along with the newest tool result, and screenshots older than the last ten are dropped from the history in batches.
|
|
171
|
+
|
|
172
|
+
## Limits
|
|
173
|
+
|
|
174
|
+
- Actions are run against a display as a whole; there is no per-window targeting.
|
|
175
|
+
- Displays are addressed one at a time. `--monitor all` captures them together but scales the result down too far to read.
|
|
176
|
+
- A rotated display falls back to the slower GDI capture, because a duplicated frame arrives in the screen's unrotated orientation.
|
|
177
|
+
- Windows blocks synthetic input to windows running as administrator, so a click on one does nothing.
|
|
178
|
+
- The Escape stop applies to `run`, which checks it before each action. Single commands have already finished by the time you could press anything.
|
|
179
|
+
- The built-in loop is covered by tests against a scripted client, not by a recorded live run.
|
|
180
|
+
|
|
181
|
+
## Development
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
pip install -e ".[dev]"
|
|
185
|
+
pytest tests -q
|
|
186
|
+
python bench/benchmark.py
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
## License
|
|
190
|
+
|
|
191
|
+
MIT
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# doppelhand
|
|
2
|
+
|
|
3
|
+
A double of you at the keyboard. doppelhand captures the Windows screen and drives the mouse and keyboard: clicking, dragging, scrolling and typing the way a person would.
|
|
4
|
+
|
|
5
|
+
Use it two ways. Give an AI agent you already have eyes and hands, so it looks at your desktop with its own model and acts through doppelhand. Or let doppelhand drive itself with Claude, using its own key.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
pip install doppelhand
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Windows only, Python 3.10 or newer. Pillow is the only dependency; screen capture and
|
|
14
|
+
input go straight through the Win32 API.
|
|
15
|
+
|
|
16
|
+
**This program moves your real mouse and types on your real keyboard.** It can click
|
|
17
|
+
anything you can click. Run it on a machine where that is acceptable and watch what it
|
|
18
|
+
does. `doppelhand run` waits for confirmation and stops when you hold Escape; the
|
|
19
|
+
single-action commands do what they are told, immediately.
|
|
20
|
+
|
|
21
|
+
## Give your agent hands
|
|
22
|
+
|
|
23
|
+
Install the skill into whichever harness you use, and it learns the loop by itself:
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
doppelhand install-skill claude-code
|
|
27
|
+
doppelhand install-skill opencode
|
|
28
|
+
doppelhand install-skill agy # global, ~/.gemini/config/skills
|
|
29
|
+
doppelhand install-skill agy-workspace # this project's .agents/skills
|
|
30
|
+
doppelhand install-skill --dest path/to/skills/doppelhand # anything else
|
|
31
|
+
doppelhand install-skill --print # read it first
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
No API key is involved. The agent takes a screenshot, reads the PNG with its own image tool, and calls back with coordinates. Every command answers with one JSON object and exits non-zero when the action did not happen.
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
$ doppelhand shot
|
|
38
|
+
{"ok": true, "path": "C:\\Users\\you\\AppData\\Local\\doppelhand\\shot.png",
|
|
39
|
+
"view": [1280, 720], "display": [1920, 1080], "scale": 0.666667, "space": "view"}
|
|
40
|
+
|
|
41
|
+
$ doppelhand click 640,360
|
|
42
|
+
{"ok": true, "action": "left_click", "at": [640, 360]}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
| Command | What it does |
|
|
46
|
+
|---|---|
|
|
47
|
+
| `shot [PATH] [--region X,Y,W,H] [--monitor N]` | capture, write a PNG, report the coordinate space |
|
|
48
|
+
| `screen` | list the displays and report the coordinate space |
|
|
49
|
+
| `click X,Y [--button] [--count] [--modifiers]` | click, double click, right click |
|
|
50
|
+
| `move X,Y` / `drag X1,Y1 X2,Y2` | move the pointer, or press and drag |
|
|
51
|
+
| `scroll up\|down\|left\|right [N] [--at X,Y]` | scroll |
|
|
52
|
+
| `type "text"` / `key "ctrl+s" [--repeat N]` | type text, press a combination |
|
|
53
|
+
| `hold "shift" 2` / `wait 1.5` | hold a key, or pause |
|
|
54
|
+
| `cursor` | where the pointer is, in both spaces |
|
|
55
|
+
|
|
56
|
+
Coordinates are the pixels of the screenshot, not of the display; doppelhand scales them back up. A point outside that view is refused rather than clamped, so a scale mistake fails loudly instead of clicking a corner. Pass `--space display` if you already have real display pixels.
|
|
57
|
+
|
|
58
|
+
Every display is addressable. `screen` lists them numbered left to right, `--monitor N` picks one, and `--monitor all` captures the lot as a single wide image. Each display has its own view space starting at 0,0, so coordinates never go negative. The display and size of the last shot are remembered, so the next command lands in the same space without repeating flags; rearranging or unplugging a monitor discards that memory rather than misplacing a click.
|
|
59
|
+
|
|
60
|
+
Screenshots include the mouse pointer, which a GDI capture leaves out by default. Turn it off with `shot --no-cursor`.
|
|
61
|
+
|
|
62
|
+
## Real time
|
|
63
|
+
|
|
64
|
+
A fresh process costs about 300ms before it does anything, which dominates a session of
|
|
65
|
+
many small actions. Run a server once and talk to it over loopback instead:
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
doppelhand serve
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
It prints a port and a secret, and writes both to `%LOCALAPPDATA%\doppelhand\serve.json`.
|
|
72
|
+
Reads are GET, anything that moves the pointer or types is POST, and arguments keep the
|
|
73
|
+
names the command line uses:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
curl -sH "X-Doppelhand-Token: $TOKEN" "http://127.0.0.1:$PORT/shot?fast=1"
|
|
77
|
+
curl -sXPOST -H "X-Doppelhand-Token: $TOKEN" "http://127.0.0.1:$PORT/click?at=640,360"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
One run of `python bench/benchmark.py` on a three-monitor desktop. The spawned column
|
|
81
|
+
moves a lot with what the machine is doing, so re-measure rather than trusting these:
|
|
82
|
+
|
|
83
|
+
| action | spawned | served | |
|
|
84
|
+
|---|---|---|---|
|
|
85
|
+
| `cursor` | 309 ms | 5.6 ms | 55× |
|
|
86
|
+
| `screen` | 262 ms | 5.3 ms | 49× |
|
|
87
|
+
| `move` | 262 ms | 5.9 ms | 44× |
|
|
88
|
+
| `shot` | 430 ms | 77 ms | 5.5× |
|
|
89
|
+
| `shot --fast` | 378 ms | 30 ms | 12× |
|
|
90
|
+
|
|
91
|
+
Most of the screenshot gain comes from capturing through DXGI Desktop Duplication, which
|
|
92
|
+
hands over the frame the compositor already holds. Opening one costs more than a whole
|
|
93
|
+
GDI capture, so it only pays inside the server; one-shot commands keep the GDI path, and
|
|
94
|
+
so does any display that refuses to be duplicated. Every shot reports which it used in
|
|
95
|
+
`source`.
|
|
96
|
+
|
|
97
|
+
The server binds to loopback only, needs the secret from that file, and turns away any
|
|
98
|
+
request that carries browser headers or names a host other than loopback.
|
|
99
|
+
|
|
100
|
+
## Or let it drive itself
|
|
101
|
+
|
|
102
|
+
This is the only path that calls the Anthropic API, the only one that needs a key, and
|
|
103
|
+
the only one that needs the SDK:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
pip install "doppelhand[run]"
|
|
107
|
+
export ANTHROPIC_API_KEY=sk-ant-...
|
|
108
|
+
doppelhand run "open the calculator and work out 19% of 4,320"
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
This loop is covered by tests against a scripted client rather than a recorded live run,
|
|
112
|
+
so treat it as the least proven part of the package.
|
|
113
|
+
|
|
114
|
+
`run` prints what it is about to do and waits for confirmation. **Hold Escape at any point to stop the run** — the key is checked before every action.
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
--max-steps N model turns before giving up (default 30)
|
|
118
|
+
--max-edge N long edge of the screenshots sent to the model (default 1280)
|
|
119
|
+
--model NAME defaults to claude-opus-5
|
|
120
|
+
-y skip the confirmation
|
|
121
|
+
-q print only the final answer
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
As a library:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from doppelhand.agent import Agent
|
|
128
|
+
|
|
129
|
+
print(Agent(max_steps=10).run("close the notification in the corner"))
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## How it works
|
|
133
|
+
|
|
134
|
+
1. `screen.py` captures a display, through GDI for a one-shot command or through DXGI Desktop Duplication when the server is holding one open, and hands back a Pillow image.
|
|
135
|
+
2. `executor.py` shrinks that image to a 1280-pixel long edge, and scales every coordinate the model returns back up to real pixels.
|
|
136
|
+
3. `inputs.py` drives the mouse and keyboard with `SendInput`. Text is typed as Unicode, so it does not depend on the active keyboard layout.
|
|
137
|
+
4. `cli.py` exposes each of those actions as a command, or `agent.py` runs the whole loop against Claude: send the screen, execute the actions in the reply, send the result, repeat.
|
|
138
|
+
|
|
139
|
+
In the built-in loop, costs are kept down two ways: one prompt-cache breakpoint moves along with the newest tool result, and screenshots older than the last ten are dropped from the history in batches.
|
|
140
|
+
|
|
141
|
+
## Limits
|
|
142
|
+
|
|
143
|
+
- Actions are run against a display as a whole; there is no per-window targeting.
|
|
144
|
+
- Displays are addressed one at a time. `--monitor all` captures them together but scales the result down too far to read.
|
|
145
|
+
- A rotated display falls back to the slower GDI capture, because a duplicated frame arrives in the screen's unrotated orientation.
|
|
146
|
+
- Windows blocks synthetic input to windows running as administrator, so a click on one does nothing.
|
|
147
|
+
- The Escape stop applies to `run`, which checks it before each action. Single commands have already finished by the time you could press anything.
|
|
148
|
+
- The built-in loop is covered by tests against a scripted client, not by a recorded live run.
|
|
149
|
+
|
|
150
|
+
## Development
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
pip install -e ".[dev]"
|
|
154
|
+
pytest tests -q
|
|
155
|
+
python bench/benchmark.py
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## License
|
|
159
|
+
|
|
160
|
+
MIT
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Measure doppelhand two ways: a fresh process per command, and a warm server.
|
|
2
|
+
|
|
3
|
+
python bench/benchmark.py
|
|
4
|
+
|
|
5
|
+
Starts its own server on a spare port, runs each action both ways, prints a table.
|
|
6
|
+
Numbers move with what the desktop is doing, so run it twice before believing a change.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import http.client
|
|
12
|
+
import json
|
|
13
|
+
import statistics
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
TOKEN = "benchmark"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def spawned(argv, rounds=5):
|
|
22
|
+
times = []
|
|
23
|
+
for _ in range(rounds):
|
|
24
|
+
start = time.perf_counter()
|
|
25
|
+
subprocess.run(argv, capture_output=True)
|
|
26
|
+
times.append((time.perf_counter() - start) * 1000)
|
|
27
|
+
return statistics.median(times)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def served(port, method, path, rounds=15):
|
|
31
|
+
times = []
|
|
32
|
+
for _ in range(rounds):
|
|
33
|
+
start = time.perf_counter()
|
|
34
|
+
connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
|
|
35
|
+
connection.request(method, path, headers={"X-Doppelhand-Token": TOKEN})
|
|
36
|
+
answer = json.loads(connection.getresponse().read())
|
|
37
|
+
connection.close()
|
|
38
|
+
times.append((time.perf_counter() - start) * 1000)
|
|
39
|
+
if not answer.get("ok"):
|
|
40
|
+
raise SystemExit(f"{path} failed: {answer}")
|
|
41
|
+
return statistics.median(times), answer
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
CASES = [
|
|
45
|
+
("cursor", ["doppelhand", "cursor"], "GET", "/cursor"),
|
|
46
|
+
("screen", ["doppelhand", "screen"], "GET", "/screen"),
|
|
47
|
+
("move", ["doppelhand", "move", "640,360"], "POST", "/move?at=640,360"),
|
|
48
|
+
("shot", ["doppelhand", "shot"], "GET", "/shot"),
|
|
49
|
+
("shot --fast", ["doppelhand", "shot", "--fast"], "GET", "/shot?fast=1"),
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def main() -> int:
|
|
54
|
+
port = 5677
|
|
55
|
+
server = subprocess.Popen(["doppelhand", "serve", "--port", str(port),
|
|
56
|
+
"--token", TOKEN, "-q"])
|
|
57
|
+
try:
|
|
58
|
+
for _ in range(50):
|
|
59
|
+
try:
|
|
60
|
+
served(port, "GET", "/health", rounds=1)
|
|
61
|
+
break
|
|
62
|
+
except OSError:
|
|
63
|
+
time.sleep(0.2)
|
|
64
|
+
else:
|
|
65
|
+
raise SystemExit("the server never came up")
|
|
66
|
+
|
|
67
|
+
served(port, "GET", "/shot", rounds=2) # let the duplication warm up
|
|
68
|
+
print(f"{'action':14}{'spawned':>12}{'served':>10}{'speedup':>10}")
|
|
69
|
+
for label, argv, method, path in CASES:
|
|
70
|
+
one_shot = spawned(argv)
|
|
71
|
+
warm, answer = served(port, method, path)
|
|
72
|
+
print(f"{label:14}{one_shot:10.1f}ms{warm:8.1f}ms{one_shot / warm:9.1f}x")
|
|
73
|
+
print(f"\nframes came from: {answer.get('source', 'n/a')}")
|
|
74
|
+
finally:
|
|
75
|
+
server.terminate()
|
|
76
|
+
server.wait(timeout=10)
|
|
77
|
+
return 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__":
|
|
81
|
+
sys.exit(main())
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "doppelhand"
|
|
7
|
+
version = "1.3.1"
|
|
8
|
+
description = "Windows agent that reads the screen and drives the mouse and keyboard like a person would"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Shanewas Ahmed", email = "shanewasahmed@gmail.com" }]
|
|
13
|
+
keywords = ["windows", "automation", "computer-use", "screenshot", "agent"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Environment :: Win32 (MS Windows)",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: Microsoft :: Windows",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Desktop Environment",
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"pillow>=10",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
# Only `doppelhand run` talks to an API. Everything else drives the desktop locally, so
|
|
33
|
+
# the SDK stays off installs that will never use it.
|
|
34
|
+
run = ["anthropic>=0.124"]
|
|
35
|
+
dev = ["pytest>=8", "anthropic>=0.124"]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/shanewas/doppelhand"
|
|
39
|
+
Issues = "https://github.com/shanewas/doppelhand/issues"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
doppelhand = "doppelhand.cli:main"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools]
|
|
45
|
+
package-dir = { "" = "src" }
|
|
46
|
+
packages = ["doppelhand"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.package-data]
|
|
49
|
+
doppelhand = ["skill/SKILL.md"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""doppelhand — read the Windows screen, drive the mouse and keyboard."""
|
|
2
|
+
|
|
3
|
+
from doppelhand.errors import (
|
|
4
|
+
ActionError,
|
|
5
|
+
Aborted,
|
|
6
|
+
DoppelhandError,
|
|
7
|
+
Refused,
|
|
8
|
+
StepLimit,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__version__ = "1.3.1"
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"ActionError",
|
|
15
|
+
"Aborted",
|
|
16
|
+
"DoppelhandError",
|
|
17
|
+
"Refused",
|
|
18
|
+
"StepLimit",
|
|
19
|
+
"__version__",
|
|
20
|
+
]
|