claude-pc-control 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- claude_pc_control-0.6.0/LICENSE +21 -0
- claude_pc_control-0.6.0/PKG-INFO +354 -0
- claude_pc_control-0.6.0/README.md +322 -0
- claude_pc_control-0.6.0/claude_pc_control.egg-info/PKG-INFO +354 -0
- claude_pc_control-0.6.0/claude_pc_control.egg-info/SOURCES.txt +51 -0
- claude_pc_control-0.6.0/claude_pc_control.egg-info/dependency_links.txt +1 -0
- claude_pc_control-0.6.0/claude_pc_control.egg-info/entry_points.txt +3 -0
- claude_pc_control-0.6.0/claude_pc_control.egg-info/requires.txt +13 -0
- claude_pc_control-0.6.0/claude_pc_control.egg-info/top_level.txt +1 -0
- claude_pc_control-0.6.0/pc_control/__init__.py +2 -0
- claude_pc_control-0.6.0/pc_control/actions.py +767 -0
- claude_pc_control-0.6.0/pc_control/cli.py +545 -0
- claude_pc_control-0.6.0/pc_control/core.py +1296 -0
- claude_pc_control-0.6.0/pc_control/cursor.py +544 -0
- claude_pc_control-0.6.0/pc_control/envvars.py +6 -0
- claude_pc_control-0.6.0/pc_control/hidden.py +144 -0
- claude_pc_control-0.6.0/pc_control/hook.py +244 -0
- claude_pc_control-0.6.0/pc_control/office.py +223 -0
- claude_pc_control-0.6.0/pc_control/perm.py +193 -0
- claude_pc_control-0.6.0/pc_control/policy.py +213 -0
- claude_pc_control-0.6.0/pc_control/server.py +812 -0
- claude_pc_control-0.6.0/pc_control/ui_text.py +87 -0
- claude_pc_control-0.6.0/pc_control/word.py +225 -0
- claude_pc_control-0.6.0/pyproject.toml +36 -0
- claude_pc_control-0.6.0/setup.cfg +4 -0
- claude_pc_control-0.6.0/tests/test_background.py +33 -0
- claude_pc_control-0.6.0/tests/test_cli.py +196 -0
- claude_pc_control-0.6.0/tests/test_cursor.py +176 -0
- claude_pc_control-0.6.0/tests/test_decide.py +30 -0
- claude_pc_control-0.6.0/tests/test_drag.py +75 -0
- claude_pc_control-0.6.0/tests/test_excel.py +70 -0
- claude_pc_control-0.6.0/tests/test_fuzz.py +129 -0
- claude_pc_control-0.6.0/tests/test_hidden.py +71 -0
- claude_pc_control-0.6.0/tests/test_hook.py +226 -0
- claude_pc_control-0.6.0/tests/test_hygiene.py +46 -0
- claude_pc_control-0.6.0/tests/test_identity.py +241 -0
- claude_pc_control-0.6.0/tests/test_input_share.py +71 -0
- claude_pc_control-0.6.0/tests/test_launcher.py +31 -0
- claude_pc_control-0.6.0/tests/test_lifecycle.py +48 -0
- claude_pc_control-0.6.0/tests/test_mark.py +54 -0
- claude_pc_control-0.6.0/tests/test_menu.py +156 -0
- claude_pc_control-0.6.0/tests/test_menu_real.py +64 -0
- claude_pc_control-0.6.0/tests/test_more.py +102 -0
- claude_pc_control-0.6.0/tests/test_policy.py +148 -0
- claude_pc_control-0.6.0/tests/test_powerpoint.py +43 -0
- claude_pc_control-0.6.0/tests/test_resize.py +113 -0
- claude_pc_control-0.6.0/tests/test_server.py +212 -0
- claude_pc_control-0.6.0/tests/test_static.py +29 -0
- claude_pc_control-0.6.0/tests/test_taskbar.py +128 -0
- claude_pc_control-0.6.0/tests/test_tuck.py +65 -0
- claude_pc_control-0.6.0/tests/test_unnamed_field.py +55 -0
- claude_pc_control-0.6.0/tests/test_web.py +58 -0
- claude_pc_control-0.6.0/tests/test_word.py +245 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 PC-Control contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: claude-pc-control
|
|
3
|
+
Version: 0.6.0
|
|
4
|
+
Summary: Let an AI see and operate Windows apps cheaply, without stealing focus or moving your mouse.
|
|
5
|
+
Author: PC-Control contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/VortexJer/PC-Control
|
|
8
|
+
Project-URL: Issues, https://github.com/VortexJer/PC-Control/issues
|
|
9
|
+
Keywords: mcp,claude,automation,windows,ui-automation
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: mcp>=1.2
|
|
19
|
+
Requires-Dist: uiautomation>=2.0
|
|
20
|
+
Requires-Dist: pywin32>=306
|
|
21
|
+
Requires-Dist: opencv-python-headless>=4.8
|
|
22
|
+
Requires-Dist: numpy>=1.26
|
|
23
|
+
Requires-Dist: Pillow>=10
|
|
24
|
+
Requires-Dist: winrt-runtime
|
|
25
|
+
Requires-Dist: winrt-Windows.Media.Ocr
|
|
26
|
+
Requires-Dist: winrt-Windows.Graphics.Imaging
|
|
27
|
+
Requires-Dist: winrt-Windows.Storage.Streams
|
|
28
|
+
Requires-Dist: winrt-Windows.Globalization
|
|
29
|
+
Requires-Dist: winrt-Windows.Foundation
|
|
30
|
+
Requires-Dist: winrt-Windows.Foundation.Collections
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# PC-Control
|
|
34
|
+
|
|
35
|
+

|
|
36
|
+
|
|
37
|
+
Let an AI see and operate Windows apps cheaply, **without stealing focus, moving your mouse or typing on your keyboard**.
|
|
38
|
+
An MCP server for Claude Code (and any MCP client). Early version: it works, but expect rough edges.
|
|
39
|
+
|
|
40
|
+
## Quick start
|
|
41
|
+
|
|
42
|
+
Windows 10/11, Python 3.10+ and [Claude Code](https://claude.com/claude-code). In PowerShell:
|
|
43
|
+
|
|
44
|
+
**Install**
|
|
45
|
+
|
|
46
|
+
```powershell
|
|
47
|
+
pip install https://github.com/VortexJer/PC-Control/archive/refs/heads/main.zip
|
|
48
|
+
pc-control install
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
**Update** (from any earlier version)
|
|
52
|
+
|
|
53
|
+
```powershell
|
|
54
|
+
pc-control update
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
**Uninstall** (everything it added, then the package)
|
|
58
|
+
|
|
59
|
+
```powershell
|
|
60
|
+
pc-control uninstall
|
|
61
|
+
pip uninstall -y claude-pc-control pc-control # whichever of the two names is installed
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Then open a **new** Claude Code session and ask, for example, *"what do I have open in Word?"*. Details in [Install](#install),
|
|
65
|
+
[Update](#update) and [Uninstall](#uninstall).
|
|
66
|
+
|
|
67
|
+
- **Sees a window without focusing it.** `look` picks the cheapest of: UI-tree text, OCR text, or an image with numbered marks.
|
|
68
|
+
It never sends empty text, text that costs more than the image, or text that does not cover the window.
|
|
69
|
+
- **Acts on a window without focusing it.** Clicks and typing go straight to the control (`BM_CLICK`, `EM_REPLACESEL`, UI Automation patterns...).
|
|
70
|
+
Nothing in the package calls `SetForegroundWindow`, `SendInput`, `keybd_event` or `SetCursorPos` (a test scans the code for them).
|
|
71
|
+
- **Answers with only what changed** after each action, not the whole screen again, and says whether the action had a visible effect.
|
|
72
|
+
- **Does what people ask for, still without focus.** Scroll, read a whole document or page, keyboard shortcuts, drop-downs, find an element (even scrolled out of view) and wait for something to appear.
|
|
73
|
+
- **Opens apps out of your way, but where you can watch.** `open_app` starts the app minimized with its button in the taskbar (no focus taken):
|
|
74
|
+
open it whenever you want to see Claude work. Its windows are marked *opened by Claude*, so the model never mistakes yours for its own.
|
|
75
|
+
- **Lets you keep working.** You and Claude can use the same app at the same time (see [You and Claude in the same app](#you-and-claude-in-the-same-app)).
|
|
76
|
+
|
|
77
|
+
Windows 10/11 only. Python 3.10+.
|
|
78
|
+
|
|
79
|
+
## Install
|
|
80
|
+
|
|
81
|
+
```powershell
|
|
82
|
+
git clone https://github.com/VortexJer/PC-Control.git
|
|
83
|
+
cd PC-Control
|
|
84
|
+
.\install.ps1 # installs the package and runs `pc-control install` (options: -Mode ask|auto|strict|bypass, -AllowReads, -Editable)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Or by hand, without cloning:
|
|
88
|
+
|
|
89
|
+
```powershell
|
|
90
|
+
pip install https://github.com/VortexJer/PC-Control/archive/refs/heads/main.zip # no git needed
|
|
91
|
+
pc-control install
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
The package is called **`claude-pc-control`** (before 0.6 it was `pc-control`; the commands `pc-control` and `pc-control-mcp` and the `pc_control`
|
|
95
|
+
code did not change). If you have an older install, do not pip-install over it: run `pc-control update` or `.\install.ps1`, which remove the old
|
|
96
|
+
package first (both install the same files, so removing the old one afterwards would delete the new code).
|
|
97
|
+
|
|
98
|
+
What `pc-control install` does (and nothing else):
|
|
99
|
+
|
|
100
|
+
| Step | Where |
|
|
101
|
+
|---|---|
|
|
102
|
+
| Registers the MCP server as **PC-Control** (`mcp__PC-Control__*`) | `claude mcp add` (scope `user` by default) |
|
|
103
|
+
| Writes the usage skill | `~/.claude/skills/pc-control/SKILL.md` |
|
|
104
|
+
| Adds a short managed block pointing to the skill | `~/.claude/CLAUDE.md` (between `<!-- pc-control:begin ... -->` and `<!-- pc-control:end -->`) |
|
|
105
|
+
| Registers the permission hooks (`PreToolUse`, `PermissionRequest`) | `~/.claude/settings.json` |
|
|
106
|
+
| Keeps a record of all of the above | `~/.claude/pc-control-install.json` |
|
|
107
|
+
| Backs up your settings before editing them (once) | `~/.claude/settings.json.pc-control-bak` |
|
|
108
|
+
|
|
109
|
+
Options: `--mode follow|auto|ask|strict|bypass` (default `follow`, see [Permission modes](#permission-modes)),
|
|
110
|
+
`--scope user|project|local`, `--allow-reads` (let `windows` and `look` run without asking).
|
|
111
|
+
Then open a **new** Claude Code session; the tools appear as `PC-Control`.
|
|
112
|
+
|
|
113
|
+
`pc-control status` shows whether it is installed, connected and paused. `pc-control pause` / `pc-control resume` is the kill switch.
|
|
114
|
+
|
|
115
|
+
## Update
|
|
116
|
+
|
|
117
|
+
From **any** earlier version (including the first one, then called `pcsight`):
|
|
118
|
+
|
|
119
|
+
```powershell
|
|
120
|
+
pc-control update # or .\update.ps1
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
It pulls the source if you installed from a git checkout, otherwise installs the latest version from GitHub (first removing the package
|
|
124
|
+
under its old name `pc-control`, if you have one: it is `claude-pc-control` since 0.6), then runs
|
|
125
|
+
`pc-control install` again with the same mode, scope and options. On the way it removes whatever an earlier version left behind
|
|
126
|
+
(the old `pcsight` server, skill, CLAUDE.md block, hooks, install record and package, and old `ask` rules for PC-Control's tools,
|
|
127
|
+
which would force a dialog on every call), and only that: your own skills, hooks, rules and notes are kept. Then open a new
|
|
128
|
+
Claude Code session.
|
|
129
|
+
|
|
130
|
+
## Uninstall
|
|
131
|
+
|
|
132
|
+
```powershell
|
|
133
|
+
pc-control uninstall # removes everything install added, and the ~/.pc-control data folder
|
|
134
|
+
pip uninstall claude-pc-control # removes the package itself (pc-control if you installed before 0.6)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
or, from the checkout, `.\uninstall.ps1` (does both; add `-KeepData` to keep `~/.pc-control`, `-KeepPackage` to keep the package).
|
|
138
|
+
|
|
139
|
+
`pc-control uninstall` is exact: it uses the record written by `install`, so it removes the MCP registration, the skill folder, the
|
|
140
|
+
managed block in `CLAUDE.md` (the file itself is deleted only if `install` created it and nothing else is in it), our hooks, any
|
|
141
|
+
read permissions it added, the settings backup and the data folder. Skills, hooks, permissions and notes that are not ours are left untouched.
|
|
142
|
+
|
|
143
|
+
## Cost
|
|
144
|
+
|
|
145
|
+
The tool definitions Claude gets on every request are about 1280 tokens for the 10 tools (the `title` that the schema generator adds to every
|
|
146
|
+
parameter, 16 % of it, is stripped). With `PC_CONTROL_TRACE=1` each action's answer ends with where its time went (`guards 4ms, cursor 270ms,
|
|
147
|
+
settle 250ms, changes 380ms`). Most of an action is the UI Automation walk that tells it what changed; it depends on how fast the app answers
|
|
148
|
+
accessibility queries.
|
|
149
|
+
|
|
150
|
+
`look` chooses by cost: it estimates what the text would cost and what an image of the window would cost (about `ceil(width/28) * ceil(height/28)`
|
|
151
|
+
tokens at the size it would send, which adapts to your monitor) and sends the cheaper one, never empty text and never text that does not
|
|
152
|
+
cover the window. The text estimate was fitted on real windows against a BPE tokenizer (within about 10 %). After an action it sends
|
|
153
|
+
only what changed. In one measured Word window, the UI-tree text cost 713 tokens against 851 for the screenshot; a minimized window is
|
|
154
|
+
read as text only. `read` gives a whole document in one call, and `read(find=...)` or `look(find=...)` only the lines or elements you need.
|
|
155
|
+
|
|
156
|
+
## Permission modes
|
|
157
|
+
|
|
158
|
+
Pick one with `pc-control install --mode ...` or the `PC_CONTROL_MODE` environment variable.
|
|
159
|
+
|
|
160
|
+
| Mode | What it does |
|
|
161
|
+
|---|---|
|
|
162
|
+
| `follow` (default) | Follows Claude Code's own permission mode, as cautious as you chose to be: `bypassPermissions` -> `bypass`, `auto` -> `auto`, `default` / `acceptEdits` / `plan` -> `ask`. |
|
|
163
|
+
| `auto` | Filters apps **by category**, no list to maintain: password managers and authenticators, terminals, system administration tools, remote access, crypto wallets, the AI client itself, mail and messaging apps, VPN/security tools, signing/identity apps, developer tools with credentials, and any window whose title is about banking, payments or passwords are *protected*: PC-Control asks you before Claude can use them. Delicate actions (pay, delete, send, install, sign out...) are asked every time. It never types into a password field. |
|
|
164
|
+
| `ask` | The lowest-risk mode: **every window** asks. A plain *Yes* is for that call only; *Yes, and don't ask again* remembers that one window for the session. Protected windows ask too. |
|
|
165
|
+
| `strict` | `auto` plus: acting only on apps listed in `PC_CONTROL_ALLOW=app1,app2` or `~/.pc-control/allow.txt`. |
|
|
166
|
+
| `bypass` | Like `--dangerously-skip-permissions`: no filters, no confirmations. Use it when you trust the session. |
|
|
167
|
+
|
|
168
|
+
In every mode: creating `~/.pc-control/PAUSE` (or `PC_CONTROL_PAUSE=1`) makes every tool refuse, and `type` needs an element id from `look` (no blind typing).
|
|
169
|
+
|
|
170
|
+
### How the questions reach you
|
|
171
|
+
|
|
172
|
+
Protected windows are **not blocked**; you decide, not the model. The model has no `confirm` flag it could set.
|
|
173
|
+
|
|
174
|
+
- **Claude Code:** **Claude Code itself shows you its permission dialog** (1. Yes / 2. Yes, and don't ask again / 3. No), and Claude
|
|
175
|
+
cannot answer it. The `PreToolUse` hook (`python -m pc_control.hook`) adds one short sentence, in the language of your Windows, as
|
|
176
|
+
the first parameter of the call (for example *question: "Claude wants to read the whole text of «Terminal» (protected window:
|
|
177
|
+
terminal or console)."*). In Claude Code's normal mode that is all; in its other modes (auto...), where Claude Code might decide
|
|
178
|
+
by itself, the hook answers `ask` so the question is always shown (the dialog then also carries a "Hook ... requires confirmation"
|
|
179
|
+
line). A plain "yes" counts once.
|
|
180
|
+
If you pick Claude Code's "Yes, and don't ask again", PC-Control narrows it to **that one window** for the rest of the session
|
|
181
|
+
(Claude Code saves a per-tool rule; the hook turns it into a per-window approval and removes the rule).
|
|
182
|
+
- The `PermissionRequest` hook answers by itself when you do not need to be bothered (a normal window outside the filter, bypass).
|
|
183
|
+
It also tells the server that a dialog really is about to be shown: once Claude Code has done that at least once, the server refuses
|
|
184
|
+
to run a protected action whose dialog was skipped (for example by a "don't ask again" rule), instead of trusting the question.
|
|
185
|
+
- **Other MCP clients** (without these hooks): a protected action simply does not run.
|
|
186
|
+
|
|
187
|
+
## What you see: Claude's cursor
|
|
188
|
+
|
|
189
|
+
When Claude clicks, an **orange pointer** glides to the element and rings it; when it types, a steady **orange I-beam** slides to the
|
|
190
|
+
insertion point. Only one is shown at a time (the one for the last action; each stays at least a moment so a quick action is visible), and it
|
|
191
|
+
moves with its window. Both are transparent click-through windows that never take focus. They fade in when Claude starts and **stay visible,
|
|
192
|
+
idle at their last position, after it finishes** (`PC_CONTROL_CURSOR=always` default; `user`: only while you are in that window; `off`).
|
|
193
|
+
Their colour adapts to what is behind them and they scale with the screen DPI.
|
|
194
|
+
|
|
195
|
+
The cursor is **isolated to the app it acts on**: it is an owned window of the target, so it follows that window's place in the stacking
|
|
196
|
+
order. **Closing the app removes them for good; minimizing it only hides them** and they come back when you restore the window.
|
|
197
|
+
When Claude Code closes, the PC-Control server exits with it and takes the cursor away.
|
|
198
|
+
|
|
199
|
+
## What is re-verified when Claude acts
|
|
200
|
+
|
|
201
|
+
An element id comes from a `look()`, but the window is alive: between `look()` and the action an id can start pointing at something else
|
|
202
|
+
(a Buy / Sell toggle that changed its label, a recycled row of a virtual list, a dialog that reused a control). So nothing is trusted from
|
|
203
|
+
the snapshot; every action re-checks, in this order, and refuses with a clear message (`NOT EXECUTED`: look again) when a step fails:
|
|
204
|
+
|
|
205
|
+
| Step | What is checked, live |
|
|
206
|
+
|---|---|
|
|
207
|
+
| Handle | The window handle still belongs to the same program that was looked at (handles are reused after a window closes). |
|
|
208
|
+
| Exists / where | The control is still there; its position is re-read. After a resize, the only replacement accepted is **one** namesake in the **same container** with the **same id** and not far away: two "OK" buttons are never guessed between. |
|
|
209
|
+
| Identity | Its live type and name equal the ones that were looked at (case, spaces and invisible direction marks ignored). Nameless controls compare their id; text fields compare type and id, so typing does not trip it. |
|
|
210
|
+
| Window policy | The window's protection category is evaluated again right before dispatching (and after the cursor's trip), and so are the dialogs it opened: a title that became a login or a payment, or a sign-in dialog that popped up, aborts the action so you are asked again. |
|
|
211
|
+
| Action policy | "Delicate action" (pay, delete, send...) and "secret field" are judged on the **live** name, id, help text and password flag, not on the snapshot. |
|
|
212
|
+
|
|
213
|
+
Also guarded, because they act without naming what they act on: clicking by `x,y` (the element of the last look under that point is judged
|
|
214
|
+
and must still be itself), `key alt+f4` and the title-bar X on a window Claude did not open (you could lose your work), the clipboard keys
|
|
215
|
+
`ctrl+c/x/v` (the clipboard may hold a copied password), and `enter` / `space` in a dialog that offers a delicate button (they press the
|
|
216
|
+
default button unseen). Fields that look like secrets (password, PIN, API key, card number, by name, id or help text) are neither typed
|
|
217
|
+
into nor read, and `open_app` asks before running interpreters and system tools (python, curl, certutil, reg...) or a script / installer
|
|
218
|
+
handed to a launcher. The text `read` returns is marked as data: instructions found in a window or web page are not yours.
|
|
219
|
+
|
|
220
|
+
Limits that remain: a click by `x,y` with **no** earlier `look()` of that window knows nothing about what is under the point (use ids);
|
|
221
|
+
OCR text has no control behind it, so it can only be checked by position; the checks protect against the model misusing PC-Control's tools,
|
|
222
|
+
not against a model that can run arbitrary commands as you (it could forge the small approval marks under `~/.pc-control/perm`: keep
|
|
223
|
+
Claude Code's own permission for shell and file writes switched on).
|
|
224
|
+
|
|
225
|
+
## Keeps working when you move or resize the window
|
|
226
|
+
|
|
227
|
+
Positions are resolved when Claude acts, not when it looked. Moving the window changes nothing (positions are relative to it); resizing
|
|
228
|
+
re-lays-out the controls, so each target is re-asked to its live control (or found again by name and type). A click with coordinates from
|
|
229
|
+
an old image is refused with a clear message instead of landing somewhere else.
|
|
230
|
+
|
|
231
|
+
Some changes cannot be absorbed: shrink a window enough and its ribbon collapses, so the button Claude was about to press no longer exists.
|
|
232
|
+
Then PC-Control does not guess: it shows a notice on that very window ("If you touch this application, Claude will not be able to act on it. ..."),
|
|
233
|
+
waits for you to put it back, and carries on by itself. If a button merely moved, PC-Control finds it again, clicks it and says so in the answer.
|
|
234
|
+
|
|
235
|
+
## You and Claude in the same app
|
|
236
|
+
|
|
237
|
+
You can keep working in the app while Claude works in it, as long as you do not hide or move its window:
|
|
238
|
+
|
|
239
|
+
- **Word:** Claude has its own insertion point (a hidden bookmark that follows the text if you edit before it). It writes there through COM
|
|
240
|
+
and the answer says in which paragraph and after which words. The formatting buttons (bold, italic, alignment...) apply to what Claude
|
|
241
|
+
writes, never to your selection. The Styles gallery (Heading 1, Normal, Strong...) works the same way: a style is Claude's own and is applied only to the new paragraph it starts (never to a paragraph of yours; if the text lands in one of yours, the answer says so). In Word `key` only offers `enter`, `space` and `backspace`, acting at Claude's own insertion point.
|
|
242
|
+
- **Your mouse:** if you are holding a mouse button (dragging, selecting), Claude waits for you to release it instead of breaking your gesture.
|
|
243
|
+
- **Your keyboard:** a key press waits until you stop typing, so it never lands in the middle of your word.
|
|
244
|
+
- **Right click** opens the app's real context menu, which closes with any click of yours, so Claude only opens it when you have been idle
|
|
245
|
+
for a few seconds and no menu is open. Menu entries are pressed through UI Automation patterns (by id, like any button).
|
|
246
|
+
|
|
247
|
+
## Tools
|
|
248
|
+
|
|
249
|
+
| Tool | What it does |
|
|
250
|
+
|---|---|
|
|
251
|
+
| `windows` | Lists windows, most recent first (`filter="word"` lists only the matching ones: 12 tokens instead of ~150); marks `[IN USE]` (where you work now), `[LAST USED]`, `[PROTECTED: ...]` / `[ASKS]`, `(minimized)`, `(hidden)` and `[mine]` (opened by Claude). A Store app is listed once. A title that matches several windows is never guessed: the answer lists them with their number. |
|
|
252
|
+
| `look` | Sees a window: UI-tree text, OCR text or an image with numbered marks, whichever is cheapest that works. Text fields show what they contain (never password fields). `find="Save|Cancel"` returns only the matching elements, also those scrolled out of view; `find="!Loading"` with `wait=10` waits until that text is gone. |
|
|
253
|
+
| `click` | Clicks an element by id or by its exact name (several with that name: the ids are listed instead of guessing), or `x,y` of the last image; `right` and `double` supported. Ids stay valid between looks. Title-bar buttons (minimize, maximize, close) and elements of apps that ignore mouse messages are pressed through UI Automation. |
|
|
254
|
+
| `type` | Types into an element id (append, or `replace`) and reads the field back to confirm the text is really there. On a drop-down or list (classic, WinForms, web `<select>`) it picks the entry with that text, and the app's change event fires. In Excel the target can be a cell address (`B3`, `Sheet2!A1`): numbers stay numbers, `=...` is a formula, and the answer says what the cell now shows. |
|
|
255
|
+
| `drag` | Holds the button and drags through points (freehand, line, rectangle, ellipse) using mouse messages: no real mouse, no focus. Apps with modern canvases may ignore it. |
|
|
256
|
+
| `key` | A key (`enter`, `tab`, `esc`, arrows, `f1`-`f12`...) or a shortcut (`ctrl+s`, `ctrl+shift+n`, `alt+f4`). Shortcuts never press real keys: PC-Control runs the app's own menu command that shows that shortcut, the element declaring it, or the text-box command (`ctrl+a/c/x/v/z`). If the app has no such command, the answer lists the shortcuts it does have (they depend on its language: in Spanish Notepad, Save is `Ctrl+G`). |
|
|
257
|
+
| `scroll` | Scrolls the window or an element by pages, or to the top/bottom, through UI Automation, scroll-bar messages or wheel messages: no focus, no mouse. Says the position before and after. |
|
|
258
|
+
| `read` | The whole text of a document, web page, editor or field, also what is scrolled out of view, in pieces (`start`, `length`), or only the lines that match (`find="a|b"`, numbered). Much cheaper than scrolling and looking. In Excel, the cells of the sheet (tab-separated, with row numbers and column letters). In PowerPoint, every slide's text, tables and speaker notes. Never reads password fields. |
|
|
259
|
+
| `show` | Shows **you** where something is: an orange circle around an element (by id, name or `x,y`) with an optional label, above its window, never clicking. It goes away when you click there, when that spot changes (you did it), or after a few seconds. For "where do I click to...?". |
|
|
260
|
+
| `open_app` | Launches an app minimized with its taskbar button, so you can open it and watch, in its own folder (as a shortcut does). If the window belongs to a process the launcher starts (Windows Media Player, Git GUI) or to a Store app, it is found and minimized too, and if it grabbed the focus it is handed back to you. Browsers (Edge, Chrome, Brave...) open behind your windows instead, not minimized, because a minimized browser hides its page from accessibility; the focus is never taken. `hidden=true` uses an invisible desktop, only when you ask for it. |
|
|
261
|
+
|
|
262
|
+
## Adapts to your PC
|
|
263
|
+
|
|
264
|
+
- Cursor and caret scale with the DPI of the window they point at (100-300 %, per monitor).
|
|
265
|
+
- Screenshots adapt to the monitor the window is on (about 55 % of its long edge, between 768 and 1344 px, never upscaled).
|
|
266
|
+
- OCR uses your Windows languages (falls back to English/Spanish, and to images if there is none).
|
|
267
|
+
- Delicate-action and sensitive-window detection knows English, Spanish, French, German, Italian and Portuguese; protected app
|
|
268
|
+
categories are matched by process name, which is language independent.
|
|
269
|
+
- Multi-monitor setups and negative coordinates are handled.
|
|
270
|
+
|
|
271
|
+
## Known limits
|
|
272
|
+
|
|
273
|
+
- A browser or Electron app that is minimized hides its page: PC-Control says so instead of answering "not found". Browsers opened with `open_app` are placed behind your windows for that reason.
|
|
274
|
+
- Excel and Word are reached through COM from their own window: Claude writes cells by address and text at its own insertion point, never through your selection. While you are editing a cell Excel refuses COM calls; PC-Control retries for a few seconds and then says so. Save (`ctrl+s`, or `ctrl+g` in a Spanish Office, where `ctrl+s` is underline) only works for files that already have a name.
|
|
275
|
+
|
|
276
|
+
- Shortcuts work only where the app exposes them as a command (a classic menu, a menu entry or element with that shortcut, or a text box). Modern menus (WinUI, WPF) only exist while open, so they are searched only for apps on the hidden desktop: opening them on your desktop could take the focus.
|
|
277
|
+
- Windows running as administrator cannot be read from a normal process (Windows blocks it).
|
|
278
|
+
- Modern Store/XAML apps (Calculator, Paint, Settings) ignore mouse messages sent to a background window. For buttons, PC-Control notices that a click had no visible effect and presses the element through UI Automation `Invoke` (what a screen reader does: no mouse, no focus), and the answer says so. A drawing canvas has no such pattern, so `drag` does nothing there (modern Paint).
|
|
279
|
+
- Store apps ignore "start minimized" and activate themselves a moment later: PC-Control minimizes them again for a few seconds and hands the focus back, but they can flash on screen for a fraction of a second.
|
|
280
|
+
- Chromium browsers (Chrome, Edge) and Electron apps expose the page contents only while their accessibility support is on; if a page shows only the browser's own controls, start it with `--force-renderer-accessibility`. Web fields are filled through UI Automation (no focus), and web buttons and checkboxes are pressed the same way.
|
|
281
|
+
- Apps without an accessibility tree rely on OCR; apps that draw everything themselves (some games) may need images.
|
|
282
|
+
- A minimized window is read through its UI tree only: no screenshot, no scrolling, no clicks by coordinates (it would have to be shown).
|
|
283
|
+
Open it from the taskbar when that is needed.
|
|
284
|
+
- Store (UWP) apps cannot run on a hidden desktop.
|
|
285
|
+
- Background typing through `WM_CHAR` is not accepted by every app (the modern Notepad ignores it); use element ids.
|
|
286
|
+
- An app that activates itself while starting can take the focus for a moment (measured: under a second) before PC-Control hands it back.
|
|
287
|
+
|
|
288
|
+
## Tests
|
|
289
|
+
|
|
290
|
+
The tests never touch your real apps: they use throw-away windows (off screen or on a hidden desktop) and a simulated Word. `test_cursor`
|
|
291
|
+
shows the orange cursor for a few seconds. Run them one by one:
|
|
292
|
+
|
|
293
|
+
```powershell
|
|
294
|
+
python tests/test_decide.py # cost/quality rules
|
|
295
|
+
python tests/test_static.py # no intrusive API calls in the package
|
|
296
|
+
python tests/test_background.py # clicks and typing without focus
|
|
297
|
+
python tests/test_hidden.py # hidden desktop: never on your screen
|
|
298
|
+
python tests/test_taskbar.py # minimized + taskbar button: never on your screen
|
|
299
|
+
python tests/test_tuck.py # when a new window is (not) sent behind
|
|
300
|
+
python tests/test_server.py # MCP server end to end, with its safety rules
|
|
301
|
+
python tests/test_policy.py # permission modes and categories
|
|
302
|
+
python tests/test_hook.py # the permission hooks
|
|
303
|
+
python tests/test_cursor.py # the orange cursor (shows it for a few seconds)
|
|
304
|
+
python tests/test_mark.py # show(): the circle is above its window and goes away when that spot changes
|
|
305
|
+
python tests/test_resize.py # moving / resizing the window mid-action
|
|
306
|
+
python tests/test_word.py # Word: Claude's own insertion point (simulated Word)
|
|
307
|
+
python tests/test_input_share.py # you and Claude working in the same app
|
|
308
|
+
python tests/test_drag.py # drag / draw
|
|
309
|
+
python tests/test_menu.py # menu entries pressed through accessibility
|
|
310
|
+
python tests/test_menu_real.py # ... against a real Windows menu
|
|
311
|
+
python tests/test_more.py # scroll, read, shortcuts, drop-downs, find and wait
|
|
312
|
+
python tests/test_excel.py # Excel: read and write cells, formulas, save (skipped without Excel)
|
|
313
|
+
python tests/test_powerpoint.py # PowerPoint: slides and notes (skipped without PowerPoint)
|
|
314
|
+
python tests/test_web.py # the same on a real web page in Edge (skipped without Edge)
|
|
315
|
+
python tests/test_unnamed_field.py # fields and buttons without an accessible name still get an id
|
|
316
|
+
python tests/test_launcher.py # a launcher whose window belongs to a child process: found, minimized, focus kept
|
|
317
|
+
python tests/test_identity.py # what is re-verified before acting (mocks + a real Buy/Sell toggle)
|
|
318
|
+
python tests/test_fuzz.py # 59 bad or odd inputs through the real protocol: clear answers, no crashes
|
|
319
|
+
python tests/test_cli.py # install / uninstall / status / pause
|
|
320
|
+
python tests/test_hygiene.py # license, README, no personal data
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
## Tested with
|
|
324
|
+
|
|
325
|
+
Besides the throw-away test windows: Word, Excel, PowerPoint, Notepad, File Explorer, Edge (web pages), Calculator, Paint, Settings,
|
|
326
|
+
KiCad, OBS Studio, RStudio, VS Code, Git GUI, Character Map, Windows Media Player and Clock, driven by Claude and by a cheaper model
|
|
327
|
+
(Haiku) as a test user, whose reports fixed most of the 0.4 issues.
|
|
328
|
+
|
|
329
|
+
## Demo
|
|
330
|
+
|
|
331
|
+
`python tools/demo.py` opens a small test window on your screen (it does not take the focus) and shows, with Claude's orange
|
|
332
|
+
cursor and a caption for each step: picking in a drop-down, scrolling, finding an entry out of view and clicking it, a shortcut run
|
|
333
|
+
through the app's menu, and reading the whole text. Then it closes the window.
|
|
334
|
+
|
|
335
|
+
## Diagnostics
|
|
336
|
+
|
|
337
|
+
`python tools/token_bench.py [other_checkout]` runs a fixed scenario through the MCP protocol and counts the tokens of the tool
|
|
338
|
+
definitions and of every answer (needs `pip install tiktoken`), to compare two versions before and after a change.
|
|
339
|
+
|
|
340
|
+
`python tools/input_monitor.py out.log stop.flag` logs focus changes, new windows, the desktop that receives input, mouse clipping and
|
|
341
|
+
capture, so you can check that PC-Control never takes your focus or input.
|
|
342
|
+
|
|
343
|
+
## License
|
|
344
|
+
|
|
345
|
+
MIT
|
|
346
|
+
|
|
347
|
+
## Disclaimer
|
|
348
|
+
|
|
349
|
+
PC-Control lets an AI assistant operate your computer. The creator of this tool is not responsible for any action Claude (or any other
|
|
350
|
+
model) takes through it, nor for its consequences. Nor is the creator responsible for bugs, errors or unexpected behavior of the tool
|
|
351
|
+
itself (a click or keystroke that lands on the wrong element, a protection that fails, lost or modified data, and so on). The permission
|
|
352
|
+
system reduces risk, it does not remove it. By using PC-Control you accept all the risks, including those of errors in the tool: review
|
|
353
|
+
what you allow, keep backups of anything important, and use it at your own risk. The software is provided "as is", without warranty of
|
|
354
|
+
any kind, as stated in the MIT license.
|