uacc 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- uacc-0.1.0/PKG-INFO +667 -0
- uacc-0.1.0/README.md +629 -0
- uacc-0.1.0/pyproject.toml +68 -0
- uacc-0.1.0/setup.cfg +4 -0
- uacc-0.1.0/tests/test_action_executor.py +150 -0
- uacc-0.1.0/tests/test_grid_encoder.py +141 -0
- uacc-0.1.0/tests/test_new_features.py +188 -0
- uacc-0.1.0/tests/test_text_map.py +212 -0
- uacc-0.1.0/uacc/__init__.py +8 -0
- uacc-0.1.0/uacc/actions/__init__.py +1 -0
- uacc-0.1.0/uacc/actions/artistic_painter.py +390 -0
- uacc-0.1.0/uacc/actions/executor.py +292 -0
- uacc-0.1.0/uacc/actions/human_mimicry.py +199 -0
- uacc-0.1.0/uacc/actions/schema.py +263 -0
- uacc-0.1.0/uacc/agent/__init__.py +1 -0
- uacc-0.1.0/uacc/agent/controller.py +356 -0
- uacc-0.1.0/uacc/agent/memory.py +238 -0
- uacc-0.1.0/uacc/agent/specialists.py +165 -0
- uacc-0.1.0/uacc/agent/verifier.py +207 -0
- uacc-0.1.0/uacc/config.py +84 -0
- uacc-0.1.0/uacc/core/__init__.py +1 -0
- uacc-0.1.0/uacc/core/accessibility.py +243 -0
- uacc-0.1.0/uacc/core/clipboard.py +176 -0
- uacc-0.1.0/uacc/core/element_finder.py +313 -0
- uacc-0.1.0/uacc/core/grid_encoder.py +244 -0
- uacc-0.1.0/uacc/core/ocr_engine.py +169 -0
- uacc-0.1.0/uacc/core/screen_capture.py +107 -0
- uacc-0.1.0/uacc/core/screen_diff.py +256 -0
- uacc-0.1.0/uacc/core/text_map.py +335 -0
- uacc-0.1.0/uacc/core/window_manager.py +587 -0
- uacc-0.1.0/uacc/models/__init__.py +1 -0
- uacc-0.1.0/uacc/models/base_adapter.py +200 -0
- uacc-0.1.0/uacc/models/hybrid_adapter.py +172 -0
- uacc-0.1.0/uacc/models/text_adapter.py +123 -0
- uacc-0.1.0/uacc/models/vision_adapter.py +140 -0
- uacc-0.1.0/uacc/web/server.py +280 -0
- uacc-0.1.0/uacc.egg-info/PKG-INFO +667 -0
- uacc-0.1.0/uacc.egg-info/SOURCES.txt +43 -0
- uacc-0.1.0/uacc.egg-info/dependency_links.txt +1 -0
- uacc-0.1.0/uacc.egg-info/entry_points.txt +3 -0
- uacc-0.1.0/uacc.egg-info/requires.txt +20 -0
- uacc-0.1.0/uacc.egg-info/top_level.txt +2 -0
- uacc-0.1.0/uacc_mcp/__init__.py +8 -0
- uacc-0.1.0/uacc_mcp/server.py +1396 -0
- uacc-0.1.0/uacc_mcp/utils.py +178 -0
uacc-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,667 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: uacc
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Universal AI Computer Control β Let any LLM control a computer with pixel-precise UI interactions
|
|
5
|
+
Author: UACC Contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/uacc-project/uacc
|
|
8
|
+
Project-URL: Repository, https://github.com/uacc-project/uacc
|
|
9
|
+
Keywords: mcp,computer-control,ai-agent,ui-automation,desktop-automation
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: mss>=9.0
|
|
20
|
+
Requires-Dist: Pillow>=10.0
|
|
21
|
+
Requires-Dist: numpy>=1.24
|
|
22
|
+
Requires-Dist: pyautogui>=0.9.54
|
|
23
|
+
Requires-Dist: pynput>=1.7
|
|
24
|
+
Requires-Dist: pywinauto>=0.6.8
|
|
25
|
+
Requires-Dist: easyocr>=1.7
|
|
26
|
+
Requires-Dist: openai>=1.0
|
|
27
|
+
Requires-Dist: pyyaml>=6.0
|
|
28
|
+
Requires-Dist: python-dotenv>=1.0
|
|
29
|
+
Requires-Dist: scikit-image>=0.21
|
|
30
|
+
Requires-Dist: mcp[cli]>=1.0
|
|
31
|
+
Requires-Dist: psutil>=5.9.0
|
|
32
|
+
Requires-Dist: fastapi>=0.100.0
|
|
33
|
+
Requires-Dist: uvicorn>=0.22.0
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
37
|
+
Requires-Dist: ruff>=0.1; extra == "dev"
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="assets/hero_banner.png" alt="UACC β Universal AI Computer Control" width="100%">
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
<h1 align="center">π₯οΈ UACC β Universal AI Computer Control</h1>
|
|
44
|
+
|
|
45
|
+
<p align="center">
|
|
46
|
+
<strong>Give any LLM the power to control a computer with pixel-precise UI interactions.</strong><br>
|
|
47
|
+
<em>Open-source β’ Works with any model β’ Vision optional β’ MCP-native</em>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<a href="https://modelcontextprotocol.io"><img src="https://img.shields.io/badge/MCP-Compatible-blue?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCI+PHBhdGggZD0iTTEyIDJMMiA3bDEwIDUgMTAtNS0xMC01eiIvPjxwYXRoIGQ9Ik0yIDE3bDEwIDUgMTAtNSIvPjxwYXRoIGQ9Ik0yIDEybDEwIDUgMTAtNSIvPjwvc3ZnPg==" alt="MCP Compatible"></a>
|
|
52
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue?style=for-the-badge&logo=python&logoColor=white" alt="Python 3.10+"></a>
|
|
53
|
+
<a href="https://github.com/uacc-project/uacc/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-green?style=for-the-badge" alt="License: MIT"></a>
|
|
54
|
+
<a href="CHANGELOG.md"><img src="https://img.shields.io/badge/version-0.1.0-orange?style=for-the-badge" alt="Version"></a>
|
|
55
|
+
<a href="https://github.com/uacc-project/uacc/stargazers"><img src="https://img.shields.io/github/stars/uacc-project/uacc?style=for-the-badge&color=yellow" alt="Stars"></a>
|
|
56
|
+
<a href="CONTRIBUTING.md"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen?style=for-the-badge" alt="PRs Welcome"></a>
|
|
57
|
+
</p>
|
|
58
|
+
|
|
59
|
+
<p align="center">
|
|
60
|
+
<a href="#-quick-start">Quick Start</a> β’
|
|
61
|
+
<a href="#-mcp-server">MCP Server</a> β’
|
|
62
|
+
<a href="#-three-modes">Three Modes</a> β’
|
|
63
|
+
<a href="#-why-uacc">Why UACC?</a> β’
|
|
64
|
+
<a href="#-architecture">Architecture</a> β’
|
|
65
|
+
<a href="#-examples">Examples</a> β’
|
|
66
|
+
<a href="CONTRIBUTING.md">Contributing</a>
|
|
67
|
+
</p>
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## π₯ The Problem
|
|
72
|
+
|
|
73
|
+
ChatGPT Work costs $200/mo. Claude Cowork is enterprise-only. Every major AI lab is building **closed-source** computer control agents locked to their own models.
|
|
74
|
+
|
|
75
|
+
**UACC is the open-source alternative.**
|
|
76
|
+
|
|
77
|
+
It works with **any LLM** β GPT-4o, Claude, Gemini, Llama, Mistral, Qwen, Phi β through a single unified interface. Your model, your rules, your data.
|
|
78
|
+
|
|
79
|
+
> **π‘ The secret weapon:** UACC doesn't even need a vision model. Text-only LLMs can "see" the screen through structured text maps with exact coordinates. Run Llama locally and control your desktop β no API costs, no cloud, no vision model required.
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## β¨ Key Features
|
|
84
|
+
|
|
85
|
+
| Feature | Description |
|
|
86
|
+
|---------|-------------|
|
|
87
|
+
| π **MCP Server** | Works with Claude Code, Claude Desktop, Hermes, OpenCode, OpenClaw, Cursor β any MCP client |
|
|
88
|
+
| π€ **Any LLM** | GPT-4o, Claude, Gemini, Llama, Mistral, Qwen, Phi β any OpenAI-compatible API |
|
|
89
|
+
| ποΈ **Vision Optional** | Text-only models get a structured "text map" of the screen with exact coordinates |
|
|
90
|
+
| π― **Pixel Precise** | Sub-10px accuracy via progressive zoom and coordinate grid overlays |
|
|
91
|
+
| π±οΈ **Human-Like** | BΓ©zier curve mouse movement, variable typing speed, natural pauses |
|
|
92
|
+
| β
**Self-Verifying** | Pre/post action verification with automatic coordinate correction |
|
|
93
|
+
| π‘οΈ **Safe Mode** | Blocks destructive actions (delete, format, rm -rf) β enabled by default |
|
|
94
|
+
| β‘ **Lightweight** | `pip install` and go. No Docker, no sandbox, no heavy infrastructure |
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## π Quick Start
|
|
99
|
+
|
|
100
|
+
Get UACC running in under 2 minutes:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
# 1. Clone & install
|
|
104
|
+
git clone https://github.com/uacc-project/uacc.git
|
|
105
|
+
cd uacc
|
|
106
|
+
pip install -e .
|
|
107
|
+
|
|
108
|
+
# 2. Configure (set your API key)
|
|
109
|
+
cp .env.example .env
|
|
110
|
+
# Edit .env β OPENAI_API_KEY=sk-...
|
|
111
|
+
|
|
112
|
+
# 3. See what a text-only LLM sees (no mouse control, safe to run)
|
|
113
|
+
python examples/demo_text_map.py
|
|
114
|
+
|
|
115
|
+
# 4. See what a vision LLM sees (saves grid overlay images)
|
|
116
|
+
python examples/demo_grid.py
|
|
117
|
+
|
|
118
|
+
# 5. Run the full agent!
|
|
119
|
+
python examples/open_notepad.py --mode hybrid
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## π MCP Server
|
|
125
|
+
|
|
126
|
+
**The fastest way to use UACC** β expose computer control as tools any AI agent can call via the [Model Context Protocol](https://modelcontextprotocol.io).
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
# Install
|
|
130
|
+
pip install -e .
|
|
131
|
+
|
|
132
|
+
# Run (stdio transport β works with Claude Code, Hermes, Cursor, etc.)
|
|
133
|
+
uacc-mcp
|
|
134
|
+
|
|
135
|
+
# Or use SSE/HTTP for remote clients
|
|
136
|
+
uacc-mcp --transport streamable-http --port 8765
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### What an AI Agent Sees
|
|
140
|
+
|
|
141
|
+
When an agent calls `get_screen_info`, it receives a structured map of every interactive element:
|
|
142
|
+
|
|
143
|
+
```
|
|
144
|
+
Screen: 1920x1080 | Window: "Visual Studio Code"
|
|
145
|
+
βββ Interactive Elements βββ
|
|
146
|
+
[e1] button "File" at (22, 15) clickable
|
|
147
|
+
[e2] button "Edit" at (67, 15) clickable
|
|
148
|
+
[e3] text_input "Search" at (680, 12) editable
|
|
149
|
+
[e15] tree_item "π src" at (110, 90) expandable (expanded)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Then it calls `click(x=22, y=15)` to click "File", or `find_element(name="Save")` to locate a button. **No vision model needed.**
|
|
153
|
+
|
|
154
|
+
### Available Tools (25 total)
|
|
155
|
+
|
|
156
|
+
| Tool | Category | Description | Example |
|
|
157
|
+
|------|----------|-------------|---------|
|
|
158
|
+
| `screenshot` | Screen Understanding | Capture full screen or a region | See what's on screen |
|
|
159
|
+
| `get_screen_info` | Screen Understanding | Get structured text map of all UI elements | Find interactive elements |
|
|
160
|
+
| `find_element` | Screen Understanding | Search for UI elements by name/type | Find the "Save" button |
|
|
161
|
+
| `get_mouse_position` | Screen Understanding | Get current cursor position | Find cursor position |
|
|
162
|
+
| `click` | Mouse & Keyboard | Click at exact coordinates | Click at (500, 300) |
|
|
163
|
+
| `click_element` | Mouse & Keyboard | Click by element name (smart targeting) | Click "Save" |
|
|
164
|
+
| `type_text` | Mouse & Keyboard | Type text via keyboard | Type text into active field |
|
|
165
|
+
| `hotkey` | Mouse & Keyboard | Press key combinations | Ctrl+S to save |
|
|
166
|
+
| `scroll` | Mouse & Keyboard | Scroll at a position | Scroll down a page |
|
|
167
|
+
| `drag` | Mouse & Keyboard | Drag from point A to B | Drag icon to folder |
|
|
168
|
+
| `hover` | Mouse & Keyboard | Move mouse and wait | Hover to trigger tooltip |
|
|
169
|
+
| `get_active_window` | Window Management | Get focused window details & state | See active window title |
|
|
170
|
+
| `list_windows` | Window Management | List all visible windows with positions | Find background windows |
|
|
171
|
+
| `focus_window` | Window Management | Bring a window to foreground by title | Bring notepad to front |
|
|
172
|
+
| `resize_window` | Window Management | Resize a window by title | Resize browser to 1280x720 |
|
|
173
|
+
| `move_window` | Window Management | Move a window by title | Move window to (0,0) |
|
|
174
|
+
| `minimize_maximize` | Window Management | Min/max/restore a window | Maximize notepad |
|
|
175
|
+
| `launch_app` | Applications | Launch an application by name or path | Launch "notepad" |
|
|
176
|
+
| `open_url` | Applications | Open a URL in default browser | Open "https://google.com" |
|
|
177
|
+
| `clipboard_read` | Clipboard | Read clipboard text content | Extract copied text |
|
|
178
|
+
| `clipboard_write` | Clipboard | Write text to clipboard | Set text for pasting |
|
|
179
|
+
| `wait_for_element` | Reliability | Poll screen until element appears | Wait for "Untitled - Notepad" |
|
|
180
|
+
| `get_action_history` | Reliability | Review recent actions | Debug agent actions |
|
|
181
|
+
| `paint_preset` | Art & Painting | Paint preset designs inside MS Paint | Paint "rose", "galaxy", "peacock" |
|
|
182
|
+
| `paint_image` | Art & Painting | Contour sketch any image inside MS Paint | Sketch outline of "my_image.png" |
|
|
183
|
+
|
|
184
|
+
### Transport Modes
|
|
185
|
+
|
|
186
|
+
| Transport | Command | Best For |
|
|
187
|
+
|-----------|---------|----------|
|
|
188
|
+
| **stdio** (default) | `uacc-mcp` | Claude Code, Claude Desktop, Cursor, Hermes, OpenClaw, OpenCode |
|
|
189
|
+
| **SSE** | `uacc-mcp --transport sse --port 8765` | Legacy remote clients |
|
|
190
|
+
| **Streamable HTTP** | `uacc-mcp --transport streamable-http --port 8765` | OpenCode (remote), OpenClaw (remote), web clients |
|
|
191
|
+
|
|
192
|
+
### Client Configuration
|
|
193
|
+
|
|
194
|
+
<details>
|
|
195
|
+
<summary><strong>Claude Code</strong></summary>
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
# Add UACC as an MCP server (project scope)
|
|
199
|
+
claude mcp add uacc -- uacc-mcp
|
|
200
|
+
|
|
201
|
+
# Or with environment variables
|
|
202
|
+
claude mcp add uacc -e UACC_SAFE_MODE=true -- uacc-mcp
|
|
203
|
+
|
|
204
|
+
# Verify
|
|
205
|
+
claude mcp list
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
For global access across all projects:
|
|
209
|
+
```bash
|
|
210
|
+
claude mcp add --scope user uacc -- uacc-mcp
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
</details>
|
|
214
|
+
|
|
215
|
+
<details>
|
|
216
|
+
<summary><strong>Claude Desktop</strong></summary>
|
|
217
|
+
|
|
218
|
+
Add to your `claude_desktop_config.json`:
|
|
219
|
+
|
|
220
|
+
```json
|
|
221
|
+
{
|
|
222
|
+
"mcpServers": {
|
|
223
|
+
"uacc": {
|
|
224
|
+
"command": "uacc-mcp",
|
|
225
|
+
"args": [],
|
|
226
|
+
"env": {
|
|
227
|
+
"UACC_SAFE_MODE": "true"
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
> **Config file location:**
|
|
235
|
+
> - **Windows:** `%APPDATA%\Claude\claude_desktop_config.json`
|
|
236
|
+
> - **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`
|
|
237
|
+
> - **Linux:** `~/.config/Claude/claude_desktop_config.json`
|
|
238
|
+
|
|
239
|
+
</details>
|
|
240
|
+
|
|
241
|
+
<details>
|
|
242
|
+
<summary><strong>Hermes</strong></summary>
|
|
243
|
+
|
|
244
|
+
Add to your `~/.hermes/config.yaml`:
|
|
245
|
+
|
|
246
|
+
```yaml
|
|
247
|
+
mcp_servers:
|
|
248
|
+
uacc:
|
|
249
|
+
command: "uacc-mcp"
|
|
250
|
+
args: []
|
|
251
|
+
env:
|
|
252
|
+
UACC_SAFE_MODE: "true"
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Or via the CLI:
|
|
256
|
+
```bash
|
|
257
|
+
hermes mcp add uacc -- uacc-mcp
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
</details>
|
|
261
|
+
|
|
262
|
+
<details>
|
|
263
|
+
<summary><strong>OpenCode</strong></summary>
|
|
264
|
+
|
|
265
|
+
Add to your `opencode.json` (project root or `~/.config/opencode/opencode.json`):
|
|
266
|
+
|
|
267
|
+
**Local (stdio):**
|
|
268
|
+
```json
|
|
269
|
+
{
|
|
270
|
+
"mcp": {
|
|
271
|
+
"uacc": {
|
|
272
|
+
"type": "local",
|
|
273
|
+
"command": ["uacc-mcp"],
|
|
274
|
+
"enabled": true
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
**Remote (streamable HTTP):**
|
|
281
|
+
```json
|
|
282
|
+
{
|
|
283
|
+
"mcp": {
|
|
284
|
+
"uacc": {
|
|
285
|
+
"type": "remote",
|
|
286
|
+
"url": "http://localhost:8765/mcp",
|
|
287
|
+
"enabled": true
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
When using remote mode, start the server with:
|
|
294
|
+
```bash
|
|
295
|
+
uacc-mcp --transport streamable-http --port 8765
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
</details>
|
|
299
|
+
|
|
300
|
+
<details>
|
|
301
|
+
<summary><strong>OpenClaw</strong></summary>
|
|
302
|
+
|
|
303
|
+
Via the CLI:
|
|
304
|
+
```bash
|
|
305
|
+
openclaw mcp add uacc -- uacc-mcp
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Or add to `~/.openclaw/openclaw.json`:
|
|
309
|
+
```json
|
|
310
|
+
{
|
|
311
|
+
"mcpServers": {
|
|
312
|
+
"uacc": {
|
|
313
|
+
"command": "uacc-mcp",
|
|
314
|
+
"args": []
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
You can also manage it through the OpenClaw Control UI at the **/mcp** tab.
|
|
321
|
+
|
|
322
|
+
</details>
|
|
323
|
+
|
|
324
|
+
<details>
|
|
325
|
+
<summary><strong>Cursor / VS Code</strong></summary>
|
|
326
|
+
|
|
327
|
+
Add to your MCP settings (`.cursor/mcp.json` or VS Code MCP config):
|
|
328
|
+
|
|
329
|
+
```json
|
|
330
|
+
{
|
|
331
|
+
"mcpServers": {
|
|
332
|
+
"uacc": {
|
|
333
|
+
"command": "uacc-mcp",
|
|
334
|
+
"args": []
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
</details>
|
|
341
|
+
|
|
342
|
+
---
|
|
343
|
+
|
|
344
|
+
## π» Standalone Web Dashboard
|
|
345
|
+
|
|
346
|
+
UACC includes a stunning, premium Web UI Dashboard to run the agent in standalone mode. It works as a full-featured desktop control panel comparable to ChatGPT Work or Claude Cowork.
|
|
347
|
+
|
|
348
|
+
### Starting the Dashboard
|
|
349
|
+
Launch the server using the shortcut console command:
|
|
350
|
+
```bash
|
|
351
|
+
uacc-ui --port 8000
|
|
352
|
+
```
|
|
353
|
+
Then navigate to `http://localhost:8000` in your web browser.
|
|
354
|
+
|
|
355
|
+
### Key Capabilities
|
|
356
|
+
- πΊ **Live Desktop Feed**: View your screen state in real-time, downsampled for low latency.
|
|
357
|
+
- πΉοΈ **Interactive Control HUD**: Type natural language commands to initiate OS automation tasks.
|
|
358
|
+
- πΌ **Job Finder Assistant**: Specialized pipeline that navigates boards, scrapes listings, and compiles structured markdown lists with direct apply links.
|
|
359
|
+
- π¬ **Research Lab**: Traces recursive queries across multiple pages to compile extensive long-form reports.
|
|
360
|
+
- π¨ **Artistic Painter Console**: Directly trigger vector drawing presets (rose, galaxy, peacock, mountains) inside Microsoft Paint.
|
|
361
|
+
- βοΈ **Config HUD**: Toggle Safe Mode, adjust iteration thresholds, and configure human-mimicry speed profiles on-the-fly.
|
|
362
|
+
|
|
363
|
+
---
|
|
364
|
+
|
|
365
|
+
## π§ Three Modes
|
|
366
|
+
|
|
367
|
+
UACC adapts to any LLM β whether it can see images or not.
|
|
368
|
+
|
|
369
|
+
### Text Mode β No Vision Required
|
|
370
|
+
|
|
371
|
+
```python
|
|
372
|
+
from uacc.agent.controller import Agent
|
|
373
|
+
|
|
374
|
+
agent = Agent(mode="text", model="llama3.1:70b", base_url="http://localhost:11434/v1")
|
|
375
|
+
result = agent.run("Open Calculator and compute 42 Γ 7")
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
The LLM receives a **structured text map** instead of a screenshot:
|
|
379
|
+
|
|
380
|
+
```
|
|
381
|
+
Screen: 1920x1080 | Window: "Calculator"
|
|
382
|
+
βββ Interactive Elements βββ
|
|
383
|
+
[e1] button "1" at (500, 600) clickable
|
|
384
|
+
[e2] button "2" at (570, 600) clickable
|
|
385
|
+
[e3] button "+" at (710, 600) clickable
|
|
386
|
+
...
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
> **π‘ This is UACC's superpower.** Any text-only LLM β even a 7B model running on a laptop β can "see" and control a desktop. No vision API costs, no cloud dependencies.
|
|
390
|
+
|
|
391
|
+
### Vision Mode
|
|
392
|
+
|
|
393
|
+
```python
|
|
394
|
+
agent = Agent(mode="vision", model="gpt-4o")
|
|
395
|
+
result = agent.run("Take a screenshot and describe what you see")
|
|
396
|
+
```
|
|
397
|
+
|
|
398
|
+
The LLM receives a screenshot with **numbered badges** on every interactive element:
|
|
399
|
+
|
|
400
|
+
<p align="center">
|
|
401
|
+
<img src="demo_grid_markers.png" alt="UACC Vision Mode β Numbered element badges overlay" width="700">
|
|
402
|
+
</p>
|
|
403
|
+
|
|
404
|
+
### Hybrid Mode (Best Accuracy)
|
|
405
|
+
|
|
406
|
+
```python
|
|
407
|
+
agent = Agent(mode="hybrid", model="gpt-4o")
|
|
408
|
+
result = agent.run("Open Notepad and type 'Hello from UACC!'")
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
Gets **both** the screenshot + text map. Cross-validates visual and textual understanding for maximum accuracy.
|
|
412
|
+
|
|
413
|
+
---
|
|
414
|
+
|
|
415
|
+
## π¨ Artistic Painting Mode
|
|
416
|
+
|
|
417
|
+
UACC features a fully dedicated vector path-tracing engine that enables AI agents to paint presets or load images from disk, convert them to edge contours, and paint them directly on screen in Microsoft Paint using smooth drag-and-drop stroke trajectories.
|
|
418
|
+
|
|
419
|
+
### Preset Paintings
|
|
420
|
+
No complex configuration needed β just invoke the `paint_preset` tool with a design name:
|
|
421
|
+
- `rose` β A mathematical Rhodonea curve
|
|
422
|
+
- `galaxy` β A double spiral galaxy pattern
|
|
423
|
+
- `mountains` β A mountain silhouette with a rising sun
|
|
424
|
+
- `peacock` β The iconic detailed peacock illustration
|
|
425
|
+
|
|
426
|
+
### Edge Tracing & Image Painting
|
|
427
|
+
Convert any image on your disk (PNG, JPEG) into vector outline drawing paths in Paint:
|
|
428
|
+
```json
|
|
429
|
+
{
|
|
430
|
+
"action": "paint_image",
|
|
431
|
+
"image_path": "C:/path/to/my_image.png",
|
|
432
|
+
"max_strokes": 150
|
|
433
|
+
}
|
|
434
|
+
```
|
|
435
|
+
UACC uses Pillow's `FIND_EDGES` filter, downsamples the outlines to prevent cluttering, and traces the lines using fast, contiguous path-tracing strokes.
|
|
436
|
+
|
|
437
|
+
---
|
|
438
|
+
|
|
439
|
+
## π€ Why UACC?
|
|
440
|
+
|
|
441
|
+
### vs. Closed-Source Solutions (Mid-2026)
|
|
442
|
+
|
|
443
|
+
| Capability | UACC (Open-Source) | ChatGPT Work (Closed) | Claude Cowork (Closed) |
|
|
444
|
+
|---|:---:|:---:|:---:|
|
|
445
|
+
| **Open Source & Extensible** | β
MIT License | β Proprietary | β Proprietary |
|
|
446
|
+
| **Any LLM Support** | β
GPT-4o, Claude, Gemini, Llama, Qwen, Phi | β GPT only | β Claude only |
|
|
447
|
+
| **Self-Hosted / Local-First** | β
Run locally (no cloud mandatory) | β Cloud-only | β Cloud-only |
|
|
448
|
+
| **Vision-Optional (Text Maps)**| β
Structured text-map feeds | β Vision required | β Vision required |
|
|
449
|
+
| **Safety Guardrails** | β
Custom safe-mode filters | β οΈ Cloud-enforced limits | β οΈ Enterprise policy |
|
|
450
|
+
| **Extensible MCP Tools** | β
Standard SSE/stdio/HTTP MCP | β No public MCP server | β No public MCP server |
|
|
451
|
+
| **Operating Cost** | β
Free (BYO API key / Local model) | π° $200/month | π° Enterprise pricing |
|
|
452
|
+
|
|
453
|
+
### vs. Open-Source Competitors
|
|
454
|
+
|
|
455
|
+
| Feature | UACC (Ours) | trycua/cua | OpenOwl | Windows MCP | UFO |
|
|
456
|
+
|---|:---:|:---:|:---:|:---:|:---:|
|
|
457
|
+
| **MCP Tools** | **23 Tools** | 25 Tools | 12 Tools | 10 Tools | N/A (Python framework) |
|
|
458
|
+
| **Vision Optional** | β
(Text Map + OCR) | β (Screenshots only) | β (Screenshots only) | β (Accessibility only) | β (Accessibility only) |
|
|
459
|
+
| **Smart Click-by-Name**| β
(Fuzzy name matching) | β οΈ (Element Index only) | β (Coordinates only) | β (Coordinates only) | β οΈ (COM API target) |
|
|
460
|
+
| **UI Polling & Wait** | β
(`wait_for_element`) | β (Manual sleeps) | β (Manual sleeps) | β (Manual sleeps) | β (Manual sleeps) |
|
|
461
|
+
| **Human Mimicry** | β
(BΓ©zier curves + typing) | β (Instant click/type) | β (Instant click/type) | β (Instant click/type) | β (Instant click/type) |
|
|
462
|
+
| **Zero VM Required** | β
(`pip install -e .`) | β (Requires Lume/VM) | β
(`pip install`) | β
(`pip install`) | β οΈ (Requires setup) |
|
|
463
|
+
| **Clipboard Support** | β
(Native Win32 & ctypes) | β
(Command wrapper) | β | β | β |
|
|
464
|
+
| **Multi-Monitor** | β
(Native resolution maps) | β οΈ (Primary display) | β | β οΈ (Primary display) | β |
|
|
465
|
+
| **Safety Mode** | β
(Regex action patterns) | β οΈ (VM-dependent) | β | β | β |
|
|
466
|
+
|
|
467
|
+
---
|
|
468
|
+
|
|
469
|
+
## ποΈ Architecture
|
|
470
|
+
|
|
471
|
+
<p align="center">
|
|
472
|
+
<img src="assets/architecture_diagram.png" alt="UACC Architecture β Observe β Think β Act β Verify" width="700">
|
|
473
|
+
</p>
|
|
474
|
+
|
|
475
|
+
### The Agent Loop
|
|
476
|
+
|
|
477
|
+
```
|
|
478
|
+
βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
479
|
+
β UACC Agent Loop β
|
|
480
|
+
β β
|
|
481
|
+
β ποΈ OBSERVE βββ π§ THINK βββ βΆοΈ ACT βββ β
VERIFY β
|
|
482
|
+
β β β β
|
|
483
|
+
β βββββββββββββββββββββββββββββββββββββββββ β
|
|
484
|
+
β β
|
|
485
|
+
β 1. Capture screen + build text map/grid overlay β
|
|
486
|
+
β 2. Send to LLM: "What action should I take?" β
|
|
487
|
+
β 3. Execute action with human-like mouse/keyboard β
|
|
488
|
+
β 4. Verify the action had the expected effect β
|
|
489
|
+
β 5. Repeat until task is complete β
|
|
490
|
+
βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
### System Architecture
|
|
494
|
+
|
|
495
|
+
```
|
|
496
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
497
|
+
β AI Agent / MCP Client β
|
|
498
|
+
β Claude Code β Hermes β OpenCode β OpenClaw β
|
|
499
|
+
β Claude Desktop β Cursor β Any MCP Client β
|
|
500
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
|
|
501
|
+
β MCP Protocol β
|
|
502
|
+
β stdio β SSE β Streamable HTTP β
|
|
503
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
|
|
504
|
+
β UACC MCP Server β
|
|
505
|
+
β screenshot β click β type β hotkey β scroll β
|
|
506
|
+
β drag β hover β find_element β get_screen_info β
|
|
507
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
|
|
508
|
+
β LLM Adapters β
|
|
509
|
+
β Text β Vision β Hybrid β
|
|
510
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
|
|
511
|
+
β UACC Core Engine β
|
|
512
|
+
β Text Map β Grid Encoder β Screen Diff β
|
|
513
|
+
β Accessibility Tree β OCR β Safe Mode β
|
|
514
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
|
|
515
|
+
β Platform Layer β
|
|
516
|
+
β pyautogui β pywinauto β EasyOCR β mss β
|
|
517
|
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
518
|
+
```
|
|
519
|
+
|
|
520
|
+
---
|
|
521
|
+
|
|
522
|
+
## π Project Structure
|
|
523
|
+
|
|
524
|
+
```
|
|
525
|
+
uacc/
|
|
526
|
+
βββ uacc/
|
|
527
|
+
β βββ config.py # Central configuration
|
|
528
|
+
β βββ core/
|
|
529
|
+
β β βββ screen_capture.py # Fast screenshot capture (mss)
|
|
530
|
+
β β βββ accessibility.py # Windows UI Automation tree
|
|
531
|
+
β β βββ ocr_engine.py # Text extraction (EasyOCR)
|
|
532
|
+
β β βββ text_map.py # Structured screen representation
|
|
533
|
+
β β βββ grid_encoder.py # Coordinate grid + markers
|
|
534
|
+
β β βββ screen_diff.py # Screen change detection
|
|
535
|
+
β βββ actions/
|
|
536
|
+
β β βββ schema.py # Action types (click, drag, type, etc.)
|
|
537
|
+
β β βββ executor.py # Real input execution (pyautogui)
|
|
538
|
+
β β βββ human_mimicry.py # BΓ©zier mouse paths, natural typing
|
|
539
|
+
β βββ models/
|
|
540
|
+
β β βββ base_adapter.py # Abstract LLM interface
|
|
541
|
+
β β βββ text_adapter.py # For text-only models
|
|
542
|
+
β β βββ vision_adapter.py # For vision models
|
|
543
|
+
β β βββ hybrid_adapter.py # Combined text + vision
|
|
544
|
+
β βββ agent/
|
|
545
|
+
β βββ controller.py # Observe β Think β Act β Verify loop
|
|
546
|
+
β βββ memory.py # Session memory & element cache
|
|
547
|
+
β βββ verifier.py # Pre/post action verification
|
|
548
|
+
βββ uacc_mcp/ # MCP Server
|
|
549
|
+
β βββ server.py # MCP tools + resources
|
|
550
|
+
β βββ utils.py # Image encoding, session state
|
|
551
|
+
βββ examples/ # Demo scripts
|
|
552
|
+
βββ tests/ # Unit tests
|
|
553
|
+
βββ pyproject.toml
|
|
554
|
+
βββ requirements.txt
|
|
555
|
+
```
|
|
556
|
+
|
|
557
|
+
---
|
|
558
|
+
|
|
559
|
+
## π Examples
|
|
560
|
+
|
|
561
|
+
```bash
|
|
562
|
+
# See what a text-only LLM "sees" (safe, no mouse control)
|
|
563
|
+
python examples/demo_text_map.py
|
|
564
|
+
|
|
565
|
+
# See what a vision LLM sees (saves grid overlay images)
|
|
566
|
+
python examples/demo_grid.py
|
|
567
|
+
|
|
568
|
+
# Full agent: open Notepad and type text
|
|
569
|
+
python examples/open_notepad.py --mode hybrid
|
|
570
|
+
|
|
571
|
+
# Full agent: search the web
|
|
572
|
+
python examples/web_search.py "What is UACC?"
|
|
573
|
+
|
|
574
|
+
# Creative: draw in MS Paint
|
|
575
|
+
python examples/draw_in_paint.py
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
---
|
|
579
|
+
|
|
580
|
+
## π§ͺ Testing
|
|
581
|
+
|
|
582
|
+
```bash
|
|
583
|
+
pip install -e ".[dev]"
|
|
584
|
+
python -m pytest tests/ -v
|
|
585
|
+
```
|
|
586
|
+
|
|
587
|
+
---
|
|
588
|
+
|
|
589
|
+
## βοΈ Configuration
|
|
590
|
+
|
|
591
|
+
All settings via environment variables or `.env` file:
|
|
592
|
+
|
|
593
|
+
```bash
|
|
594
|
+
# LLM
|
|
595
|
+
OPENAI_API_KEY=sk-... # Your API key
|
|
596
|
+
OPENAI_MODEL=gpt-4o # Model name
|
|
597
|
+
OPENAI_BASE_URL= # Custom endpoint (Ollama, vLLM, etc.)
|
|
598
|
+
|
|
599
|
+
# UACC
|
|
600
|
+
UACC_MODE=hybrid # text | vision | hybrid
|
|
601
|
+
UACC_SAFE_MODE=true # Block destructive actions
|
|
602
|
+
UACC_HUMAN_MIMICRY=true # BΓ©zier curve mouse movement
|
|
603
|
+
UACC_MAX_ITERATIONS=30 # Max agent iterations per task
|
|
604
|
+
UACC_GRID_MODE=medium # coarse | medium | fine | micro
|
|
605
|
+
```
|
|
606
|
+
|
|
607
|
+
See [`.env.example`](.env.example) for the full list.
|
|
608
|
+
|
|
609
|
+
---
|
|
610
|
+
|
|
611
|
+
## β οΈ Safety
|
|
612
|
+
|
|
613
|
+
UACC controls your **real mouse and keyboard**. Built-in protections:
|
|
614
|
+
|
|
615
|
+
- π‘οΈ **Safe Mode** (on by default) β blocks destructive patterns (`delete`, `rm -rf`, `format`, etc.)
|
|
616
|
+
- π±οΈ **pyautogui failsafe** β move your mouse to any screen corner to instantly abort
|
|
617
|
+
- π **Action logging** β every action is logged with coordinates and reasoning
|
|
618
|
+
- π **MCP boundaries** β the MCP server only exposes UI tools, no file/shell/network access
|
|
619
|
+
|
|
620
|
+
> **Tip:** Start with `demo_text_map.py` and `demo_grid.py` β they're read-only and don't control the mouse.
|
|
621
|
+
|
|
622
|
+
---
|
|
623
|
+
|
|
624
|
+
## πΊοΈ Roadmap
|
|
625
|
+
|
|
626
|
+
- [ ] π§ Linux support (X11/Wayland accessibility)
|
|
627
|
+
- [ ] π macOS support (AppleScript + Accessibility API)
|
|
628
|
+
- [ ] π Browser-specific adapter (Playwright/Selenium integration)
|
|
629
|
+
- [ ] πΈ Video recording of agent sessions
|
|
630
|
+
- [ ] π Multi-monitor support
|
|
631
|
+
- [ ] π Action replay and debugging UI
|
|
632
|
+
- [ ] π§© Plugin system for custom actions
|
|
633
|
+
- [ ] ποΈ Benchmark suite for agent evaluation
|
|
634
|
+
|
|
635
|
+
---
|
|
636
|
+
|
|
637
|
+
## π€ Contributing
|
|
638
|
+
|
|
639
|
+
We welcome contributions of all kinds! See our [Contributing Guide](CONTRIBUTING.md) for details.
|
|
640
|
+
|
|
641
|
+
- π [Report a bug](../../issues/new?template=bug_report.md)
|
|
642
|
+
- β¨ [Request a feature](../../issues/new?template=feature_request.md)
|
|
643
|
+
- π Improve documentation
|
|
644
|
+
- π§ͺ Add test coverage
|
|
645
|
+
|
|
646
|
+
**Good first issues** are labeled and waiting for you β check the [issues page](../../issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22).
|
|
647
|
+
|
|
648
|
+
---
|
|
649
|
+
|
|
650
|
+
## π License
|
|
651
|
+
|
|
652
|
+
[MIT](https://github.com/uacc-project/uacc/blob/main/LICENSE) β use it, fork it, ship it.
|
|
653
|
+
|
|
654
|
+
---
|
|
655
|
+
|
|
656
|
+
## β Star History
|
|
657
|
+
|
|
658
|
+
If UACC is useful to you, consider giving it a star! It helps others discover the project.
|
|
659
|
+
|
|
660
|
+
[](https://star-history.com/#uacc-project/uacc&Date)
|
|
661
|
+
|
|
662
|
+
---
|
|
663
|
+
|
|
664
|
+
<p align="center">
|
|
665
|
+
<strong>Built for the agentic era.</strong><br>
|
|
666
|
+
<em>UACC β because AI should work with any model, on any desktop, under your control.</em>
|
|
667
|
+
</p>
|