uacc 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. uacc-0.1.0/PKG-INFO +667 -0
  2. uacc-0.1.0/README.md +629 -0
  3. uacc-0.1.0/pyproject.toml +68 -0
  4. uacc-0.1.0/setup.cfg +4 -0
  5. uacc-0.1.0/tests/test_action_executor.py +150 -0
  6. uacc-0.1.0/tests/test_grid_encoder.py +141 -0
  7. uacc-0.1.0/tests/test_new_features.py +188 -0
  8. uacc-0.1.0/tests/test_text_map.py +212 -0
  9. uacc-0.1.0/uacc/__init__.py +8 -0
  10. uacc-0.1.0/uacc/actions/__init__.py +1 -0
  11. uacc-0.1.0/uacc/actions/artistic_painter.py +390 -0
  12. uacc-0.1.0/uacc/actions/executor.py +292 -0
  13. uacc-0.1.0/uacc/actions/human_mimicry.py +199 -0
  14. uacc-0.1.0/uacc/actions/schema.py +263 -0
  15. uacc-0.1.0/uacc/agent/__init__.py +1 -0
  16. uacc-0.1.0/uacc/agent/controller.py +356 -0
  17. uacc-0.1.0/uacc/agent/memory.py +238 -0
  18. uacc-0.1.0/uacc/agent/specialists.py +165 -0
  19. uacc-0.1.0/uacc/agent/verifier.py +207 -0
  20. uacc-0.1.0/uacc/config.py +84 -0
  21. uacc-0.1.0/uacc/core/__init__.py +1 -0
  22. uacc-0.1.0/uacc/core/accessibility.py +243 -0
  23. uacc-0.1.0/uacc/core/clipboard.py +176 -0
  24. uacc-0.1.0/uacc/core/element_finder.py +313 -0
  25. uacc-0.1.0/uacc/core/grid_encoder.py +244 -0
  26. uacc-0.1.0/uacc/core/ocr_engine.py +169 -0
  27. uacc-0.1.0/uacc/core/screen_capture.py +107 -0
  28. uacc-0.1.0/uacc/core/screen_diff.py +256 -0
  29. uacc-0.1.0/uacc/core/text_map.py +335 -0
  30. uacc-0.1.0/uacc/core/window_manager.py +587 -0
  31. uacc-0.1.0/uacc/models/__init__.py +1 -0
  32. uacc-0.1.0/uacc/models/base_adapter.py +200 -0
  33. uacc-0.1.0/uacc/models/hybrid_adapter.py +172 -0
  34. uacc-0.1.0/uacc/models/text_adapter.py +123 -0
  35. uacc-0.1.0/uacc/models/vision_adapter.py +140 -0
  36. uacc-0.1.0/uacc/web/server.py +280 -0
  37. uacc-0.1.0/uacc.egg-info/PKG-INFO +667 -0
  38. uacc-0.1.0/uacc.egg-info/SOURCES.txt +43 -0
  39. uacc-0.1.0/uacc.egg-info/dependency_links.txt +1 -0
  40. uacc-0.1.0/uacc.egg-info/entry_points.txt +3 -0
  41. uacc-0.1.0/uacc.egg-info/requires.txt +20 -0
  42. uacc-0.1.0/uacc.egg-info/top_level.txt +2 -0
  43. uacc-0.1.0/uacc_mcp/__init__.py +8 -0
  44. uacc-0.1.0/uacc_mcp/server.py +1396 -0
  45. uacc-0.1.0/uacc_mcp/utils.py +178 -0
uacc-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,667 @@
1
+ Metadata-Version: 2.4
2
+ Name: uacc
3
+ Version: 0.1.0
4
+ Summary: Universal AI Computer Control β€” Let any LLM control a computer with pixel-precise UI interactions
5
+ Author: UACC Contributors
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/uacc-project/uacc
8
+ Project-URL: Repository, https://github.com/uacc-project/uacc
9
+ Keywords: mcp,computer-control,ai-agent,ui-automation,desktop-automation
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.10
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Operating System :: Microsoft :: Windows
15
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
16
+ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
17
+ Requires-Python: >=3.10
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: mss>=9.0
20
+ Requires-Dist: Pillow>=10.0
21
+ Requires-Dist: numpy>=1.24
22
+ Requires-Dist: pyautogui>=0.9.54
23
+ Requires-Dist: pynput>=1.7
24
+ Requires-Dist: pywinauto>=0.6.8
25
+ Requires-Dist: easyocr>=1.7
26
+ Requires-Dist: openai>=1.0
27
+ Requires-Dist: pyyaml>=6.0
28
+ Requires-Dist: python-dotenv>=1.0
29
+ Requires-Dist: scikit-image>=0.21
30
+ Requires-Dist: mcp[cli]>=1.0
31
+ Requires-Dist: psutil>=5.9.0
32
+ Requires-Dist: fastapi>=0.100.0
33
+ Requires-Dist: uvicorn>=0.22.0
34
+ Provides-Extra: dev
35
+ Requires-Dist: pytest>=7.0; extra == "dev"
36
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
37
+ Requires-Dist: ruff>=0.1; extra == "dev"
38
+
39
+ <p align="center">
40
+ <img src="assets/hero_banner.png" alt="UACC β€” Universal AI Computer Control" width="100%">
41
+ </p>
42
+
43
+ <h1 align="center">πŸ–₯️ UACC β€” Universal AI Computer Control</h1>
44
+
45
+ <p align="center">
46
+ <strong>Give any LLM the power to control a computer with pixel-precise UI interactions.</strong><br>
47
+ <em>Open-source β€’ Works with any model β€’ Vision optional β€’ MCP-native</em>
48
+ </p>
49
+
50
+ <p align="center">
51
+ <a href="https://modelcontextprotocol.io"><img src="https://img.shields.io/badge/MCP-Compatible-blue?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCI+PHBhdGggZD0iTTEyIDJMMiA3bDEwIDUgMTAtNS0xMC01eiIvPjxwYXRoIGQ9Ik0yIDE3bDEwIDUgMTAtNSIvPjxwYXRoIGQ9Ik0yIDEybDEwIDUgMTAtNSIvPjwvc3ZnPg==" alt="MCP Compatible"></a>
52
+ <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue?style=for-the-badge&logo=python&logoColor=white" alt="Python 3.10+"></a>
53
+ <a href="https://github.com/uacc-project/uacc/blob/main/LICENSE"><img src="https://img.shields.io/badge/License-MIT-green?style=for-the-badge" alt="License: MIT"></a>
54
+ <a href="CHANGELOG.md"><img src="https://img.shields.io/badge/version-0.1.0-orange?style=for-the-badge" alt="Version"></a>
55
+ <a href="https://github.com/uacc-project/uacc/stargazers"><img src="https://img.shields.io/github/stars/uacc-project/uacc?style=for-the-badge&color=yellow" alt="Stars"></a>
56
+ <a href="CONTRIBUTING.md"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen?style=for-the-badge" alt="PRs Welcome"></a>
57
+ </p>
58
+
59
+ <p align="center">
60
+ <a href="#-quick-start">Quick Start</a> β€’
61
+ <a href="#-mcp-server">MCP Server</a> β€’
62
+ <a href="#-three-modes">Three Modes</a> β€’
63
+ <a href="#-why-uacc">Why UACC?</a> β€’
64
+ <a href="#-architecture">Architecture</a> β€’
65
+ <a href="#-examples">Examples</a> β€’
66
+ <a href="CONTRIBUTING.md">Contributing</a>
67
+ </p>
68
+
69
+ ---
70
+
71
+ ## πŸ”₯ The Problem
72
+
73
+ ChatGPT Work costs $200/mo. Claude Cowork is enterprise-only. Every major AI lab is building **closed-source** computer control agents locked to their own models.
74
+
75
+ **UACC is the open-source alternative.**
76
+
77
+ It works with **any LLM** β€” GPT-4o, Claude, Gemini, Llama, Mistral, Qwen, Phi β€” through a single unified interface. Your model, your rules, your data.
78
+
79
+ > **πŸ’‘ The secret weapon:** UACC doesn't even need a vision model. Text-only LLMs can "see" the screen through structured text maps with exact coordinates. Run Llama locally and control your desktop β€” no API costs, no cloud, no vision model required.
80
+
81
+ ---
82
+
83
+ ## ✨ Key Features
84
+
85
+ | Feature | Description |
86
+ |---------|-------------|
87
+ | πŸ”Œ **MCP Server** | Works with Claude Code, Claude Desktop, Hermes, OpenCode, OpenClaw, Cursor β€” any MCP client |
88
+ | πŸ€– **Any LLM** | GPT-4o, Claude, Gemini, Llama, Mistral, Qwen, Phi β€” any OpenAI-compatible API |
89
+ | πŸ‘οΈ **Vision Optional** | Text-only models get a structured "text map" of the screen with exact coordinates |
90
+ | 🎯 **Pixel Precise** | Sub-10px accuracy via progressive zoom and coordinate grid overlays |
91
+ | πŸ–±οΈ **Human-Like** | BΓ©zier curve mouse movement, variable typing speed, natural pauses |
92
+ | βœ… **Self-Verifying** | Pre/post action verification with automatic coordinate correction |
93
+ | πŸ›‘οΈ **Safe Mode** | Blocks destructive actions (delete, format, rm -rf) β€” enabled by default |
94
+ | ⚑ **Lightweight** | `pip install` and go. No Docker, no sandbox, no heavy infrastructure |
95
+
96
+ ---
97
+
98
+ ## πŸš€ Quick Start
99
+
100
+ Get UACC running in under 2 minutes:
101
+
102
+ ```bash
103
+ # 1. Clone & install
104
+ git clone https://github.com/uacc-project/uacc.git
105
+ cd uacc
106
+ pip install -e .
107
+
108
+ # 2. Configure (set your API key)
109
+ cp .env.example .env
110
+ # Edit .env β†’ OPENAI_API_KEY=sk-...
111
+
112
+ # 3. See what a text-only LLM sees (no mouse control, safe to run)
113
+ python examples/demo_text_map.py
114
+
115
+ # 4. See what a vision LLM sees (saves grid overlay images)
116
+ python examples/demo_grid.py
117
+
118
+ # 5. Run the full agent!
119
+ python examples/open_notepad.py --mode hybrid
120
+ ```
121
+
122
+ ---
123
+
124
+ ## πŸ”Œ MCP Server
125
+
126
+ **The fastest way to use UACC** β€” expose computer control as tools any AI agent can call via the [Model Context Protocol](https://modelcontextprotocol.io).
127
+
128
+ ```bash
129
+ # Install
130
+ pip install -e .
131
+
132
+ # Run (stdio transport β€” works with Claude Code, Hermes, Cursor, etc.)
133
+ uacc-mcp
134
+
135
+ # Or use SSE/HTTP for remote clients
136
+ uacc-mcp --transport streamable-http --port 8765
137
+ ```
138
+
139
+ ### What an AI Agent Sees
140
+
141
+ When an agent calls `get_screen_info`, it receives a structured map of every interactive element:
142
+
143
+ ```
144
+ Screen: 1920x1080 | Window: "Visual Studio Code"
145
+ ─── Interactive Elements ───
146
+ [e1] button "File" at (22, 15) clickable
147
+ [e2] button "Edit" at (67, 15) clickable
148
+ [e3] text_input "Search" at (680, 12) editable
149
+ [e15] tree_item "πŸ“ src" at (110, 90) expandable (expanded)
150
+ ```
151
+
152
+ Then it calls `click(x=22, y=15)` to click "File", or `find_element(name="Save")` to locate a button. **No vision model needed.**
153
+
154
+ ### Available Tools (25 total)
155
+
156
+ | Tool | Category | Description | Example |
157
+ |------|----------|-------------|---------|
158
+ | `screenshot` | Screen Understanding | Capture full screen or a region | See what's on screen |
159
+ | `get_screen_info` | Screen Understanding | Get structured text map of all UI elements | Find interactive elements |
160
+ | `find_element` | Screen Understanding | Search for UI elements by name/type | Find the "Save" button |
161
+ | `get_mouse_position` | Screen Understanding | Get current cursor position | Find cursor position |
162
+ | `click` | Mouse & Keyboard | Click at exact coordinates | Click at (500, 300) |
163
+ | `click_element` | Mouse & Keyboard | Click by element name (smart targeting) | Click "Save" |
164
+ | `type_text` | Mouse & Keyboard | Type text via keyboard | Type text into active field |
165
+ | `hotkey` | Mouse & Keyboard | Press key combinations | Ctrl+S to save |
166
+ | `scroll` | Mouse & Keyboard | Scroll at a position | Scroll down a page |
167
+ | `drag` | Mouse & Keyboard | Drag from point A to B | Drag icon to folder |
168
+ | `hover` | Mouse & Keyboard | Move mouse and wait | Hover to trigger tooltip |
169
+ | `get_active_window` | Window Management | Get focused window details & state | See active window title |
170
+ | `list_windows` | Window Management | List all visible windows with positions | Find background windows |
171
+ | `focus_window` | Window Management | Bring a window to foreground by title | Bring notepad to front |
172
+ | `resize_window` | Window Management | Resize a window by title | Resize browser to 1280x720 |
173
+ | `move_window` | Window Management | Move a window by title | Move window to (0,0) |
174
+ | `minimize_maximize` | Window Management | Min/max/restore a window | Maximize notepad |
175
+ | `launch_app` | Applications | Launch an application by name or path | Launch "notepad" |
176
+ | `open_url` | Applications | Open a URL in default browser | Open "https://google.com" |
177
+ | `clipboard_read` | Clipboard | Read clipboard text content | Extract copied text |
178
+ | `clipboard_write` | Clipboard | Write text to clipboard | Set text for pasting |
179
+ | `wait_for_element` | Reliability | Poll screen until element appears | Wait for "Untitled - Notepad" |
180
+ | `get_action_history` | Reliability | Review recent actions | Debug agent actions |
181
+ | `paint_preset` | Art & Painting | Paint preset designs inside MS Paint | Paint "rose", "galaxy", "peacock" |
182
+ | `paint_image` | Art & Painting | Contour sketch any image inside MS Paint | Sketch outline of "my_image.png" |
183
+
184
+ ### Transport Modes
185
+
186
+ | Transport | Command | Best For |
187
+ |-----------|---------|----------|
188
+ | **stdio** (default) | `uacc-mcp` | Claude Code, Claude Desktop, Cursor, Hermes, OpenClaw, OpenCode |
189
+ | **SSE** | `uacc-mcp --transport sse --port 8765` | Legacy remote clients |
190
+ | **Streamable HTTP** | `uacc-mcp --transport streamable-http --port 8765` | OpenCode (remote), OpenClaw (remote), web clients |
191
+
192
+ ### Client Configuration
193
+
194
+ <details>
195
+ <summary><strong>Claude Code</strong></summary>
196
+
197
+ ```bash
198
+ # Add UACC as an MCP server (project scope)
199
+ claude mcp add uacc -- uacc-mcp
200
+
201
+ # Or with environment variables
202
+ claude mcp add uacc -e UACC_SAFE_MODE=true -- uacc-mcp
203
+
204
+ # Verify
205
+ claude mcp list
206
+ ```
207
+
208
+ For global access across all projects:
209
+ ```bash
210
+ claude mcp add --scope user uacc -- uacc-mcp
211
+ ```
212
+
213
+ </details>
214
+
215
+ <details>
216
+ <summary><strong>Claude Desktop</strong></summary>
217
+
218
+ Add to your `claude_desktop_config.json`:
219
+
220
+ ```json
221
+ {
222
+ "mcpServers": {
223
+ "uacc": {
224
+ "command": "uacc-mcp",
225
+ "args": [],
226
+ "env": {
227
+ "UACC_SAFE_MODE": "true"
228
+ }
229
+ }
230
+ }
231
+ }
232
+ ```
233
+
234
+ > **Config file location:**
235
+ > - **Windows:** `%APPDATA%\Claude\claude_desktop_config.json`
236
+ > - **macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json`
237
+ > - **Linux:** `~/.config/Claude/claude_desktop_config.json`
238
+
239
+ </details>
240
+
241
+ <details>
242
+ <summary><strong>Hermes</strong></summary>
243
+
244
+ Add to your `~/.hermes/config.yaml`:
245
+
246
+ ```yaml
247
+ mcp_servers:
248
+ uacc:
249
+ command: "uacc-mcp"
250
+ args: []
251
+ env:
252
+ UACC_SAFE_MODE: "true"
253
+ ```
254
+
255
+ Or via the CLI:
256
+ ```bash
257
+ hermes mcp add uacc -- uacc-mcp
258
+ ```
259
+
260
+ </details>
261
+
262
+ <details>
263
+ <summary><strong>OpenCode</strong></summary>
264
+
265
+ Add to your `opencode.json` (project root or `~/.config/opencode/opencode.json`):
266
+
267
+ **Local (stdio):**
268
+ ```json
269
+ {
270
+ "mcp": {
271
+ "uacc": {
272
+ "type": "local",
273
+ "command": ["uacc-mcp"],
274
+ "enabled": true
275
+ }
276
+ }
277
+ }
278
+ ```
279
+
280
+ **Remote (streamable HTTP):**
281
+ ```json
282
+ {
283
+ "mcp": {
284
+ "uacc": {
285
+ "type": "remote",
286
+ "url": "http://localhost:8765/mcp",
287
+ "enabled": true
288
+ }
289
+ }
290
+ }
291
+ ```
292
+
293
+ When using remote mode, start the server with:
294
+ ```bash
295
+ uacc-mcp --transport streamable-http --port 8765
296
+ ```
297
+
298
+ </details>
299
+
300
+ <details>
301
+ <summary><strong>OpenClaw</strong></summary>
302
+
303
+ Via the CLI:
304
+ ```bash
305
+ openclaw mcp add uacc -- uacc-mcp
306
+ ```
307
+
308
+ Or add to `~/.openclaw/openclaw.json`:
309
+ ```json
310
+ {
311
+ "mcpServers": {
312
+ "uacc": {
313
+ "command": "uacc-mcp",
314
+ "args": []
315
+ }
316
+ }
317
+ }
318
+ ```
319
+
320
+ You can also manage it through the OpenClaw Control UI at the **/mcp** tab.
321
+
322
+ </details>
323
+
324
+ <details>
325
+ <summary><strong>Cursor / VS Code</strong></summary>
326
+
327
+ Add to your MCP settings (`.cursor/mcp.json` or VS Code MCP config):
328
+
329
+ ```json
330
+ {
331
+ "mcpServers": {
332
+ "uacc": {
333
+ "command": "uacc-mcp",
334
+ "args": []
335
+ }
336
+ }
337
+ }
338
+ ```
339
+
340
+ </details>
341
+
342
+ ---
343
+
344
+ ## πŸ’» Standalone Web Dashboard
345
+
346
+ UACC includes a stunning, premium Web UI Dashboard to run the agent in standalone mode. It works as a full-featured desktop control panel comparable to ChatGPT Work or Claude Cowork.
347
+
348
+ ### Starting the Dashboard
349
+ Launch the server using the shortcut console command:
350
+ ```bash
351
+ uacc-ui --port 8000
352
+ ```
353
+ Then navigate to `http://localhost:8000` in your web browser.
354
+
355
+ ### Key Capabilities
356
+ - πŸ“Ί **Live Desktop Feed**: View your screen state in real-time, downsampled for low latency.
357
+ - πŸ•ΉοΈ **Interactive Control HUD**: Type natural language commands to initiate OS automation tasks.
358
+ - πŸ’Ό **Job Finder Assistant**: Specialized pipeline that navigates boards, scrapes listings, and compiles structured markdown lists with direct apply links.
359
+ - πŸ”¬ **Research Lab**: Traces recursive queries across multiple pages to compile extensive long-form reports.
360
+ - 🎨 **Artistic Painter Console**: Directly trigger vector drawing presets (rose, galaxy, peacock, mountains) inside Microsoft Paint.
361
+ - βš™οΈ **Config HUD**: Toggle Safe Mode, adjust iteration thresholds, and configure human-mimicry speed profiles on-the-fly.
362
+
363
+ ---
364
+
365
+ ## 🧠 Three Modes
366
+
367
+ UACC adapts to any LLM β€” whether it can see images or not.
368
+
369
+ ### Text Mode β€” No Vision Required
370
+
371
+ ```python
372
+ from uacc.agent.controller import Agent
373
+
374
+ agent = Agent(mode="text", model="llama3.1:70b", base_url="http://localhost:11434/v1")
375
+ result = agent.run("Open Calculator and compute 42 Γ— 7")
376
+ ```
377
+
378
+ The LLM receives a **structured text map** instead of a screenshot:
379
+
380
+ ```
381
+ Screen: 1920x1080 | Window: "Calculator"
382
+ ─── Interactive Elements ───
383
+ [e1] button "1" at (500, 600) clickable
384
+ [e2] button "2" at (570, 600) clickable
385
+ [e3] button "+" at (710, 600) clickable
386
+ ...
387
+ ```
388
+
389
+ > **πŸ’‘ This is UACC's superpower.** Any text-only LLM β€” even a 7B model running on a laptop β€” can "see" and control a desktop. No vision API costs, no cloud dependencies.
390
+
391
+ ### Vision Mode
392
+
393
+ ```python
394
+ agent = Agent(mode="vision", model="gpt-4o")
395
+ result = agent.run("Take a screenshot and describe what you see")
396
+ ```
397
+
398
+ The LLM receives a screenshot with **numbered badges** on every interactive element:
399
+
400
+ <p align="center">
401
+ <img src="demo_grid_markers.png" alt="UACC Vision Mode β€” Numbered element badges overlay" width="700">
402
+ </p>
403
+
404
+ ### Hybrid Mode (Best Accuracy)
405
+
406
+ ```python
407
+ agent = Agent(mode="hybrid", model="gpt-4o")
408
+ result = agent.run("Open Notepad and type 'Hello from UACC!'")
409
+ ```
410
+
411
+ Gets **both** the screenshot + text map. Cross-validates visual and textual understanding for maximum accuracy.
412
+
413
+ ---
414
+
415
+ ## 🎨 Artistic Painting Mode
416
+
417
+ UACC features a fully dedicated vector path-tracing engine that enables AI agents to paint presets or load images from disk, convert them to edge contours, and paint them directly on screen in Microsoft Paint using smooth drag-and-drop stroke trajectories.
418
+
419
+ ### Preset Paintings
420
+ No complex configuration needed β€” just invoke the `paint_preset` tool with a design name:
421
+ - `rose` β€” A mathematical Rhodonea curve
422
+ - `galaxy` β€” A double spiral galaxy pattern
423
+ - `mountains` β€” A mountain silhouette with a rising sun
424
+ - `peacock` β€” The iconic detailed peacock illustration
425
+
426
+ ### Edge Tracing & Image Painting
427
+ Convert any image on your disk (PNG, JPEG) into vector outline drawing paths in Paint:
428
+ ```json
429
+ {
430
+ "action": "paint_image",
431
+ "image_path": "C:/path/to/my_image.png",
432
+ "max_strokes": 150
433
+ }
434
+ ```
435
+ UACC uses Pillow's `FIND_EDGES` filter, downsamples the outlines to prevent cluttering, and traces the lines using fast, contiguous path-tracing strokes.
436
+
437
+ ---
438
+
439
+ ## πŸ€” Why UACC?
440
+
441
+ ### vs. Closed-Source Solutions (Mid-2026)
442
+
443
+ | Capability | UACC (Open-Source) | ChatGPT Work (Closed) | Claude Cowork (Closed) |
444
+ |---|:---:|:---:|:---:|
445
+ | **Open Source & Extensible** | βœ… MIT License | ❌ Proprietary | ❌ Proprietary |
446
+ | **Any LLM Support** | βœ… GPT-4o, Claude, Gemini, Llama, Qwen, Phi | ❌ GPT only | ❌ Claude only |
447
+ | **Self-Hosted / Local-First** | βœ… Run locally (no cloud mandatory) | ❌ Cloud-only | ❌ Cloud-only |
448
+ | **Vision-Optional (Text Maps)**| βœ… Structured text-map feeds | ❌ Vision required | ❌ Vision required |
449
+ | **Safety Guardrails** | βœ… Custom safe-mode filters | ⚠️ Cloud-enforced limits | ⚠️ Enterprise policy |
450
+ | **Extensible MCP Tools** | βœ… Standard SSE/stdio/HTTP MCP | ❌ No public MCP server | ❌ No public MCP server |
451
+ | **Operating Cost** | βœ… Free (BYO API key / Local model) | πŸ’° $200/month | πŸ’° Enterprise pricing |
452
+
453
+ ### vs. Open-Source Competitors
454
+
455
+ | Feature | UACC (Ours) | trycua/cua | OpenOwl | Windows MCP | UFO |
456
+ |---|:---:|:---:|:---:|:---:|:---:|
457
+ | **MCP Tools** | **23 Tools** | 25 Tools | 12 Tools | 10 Tools | N/A (Python framework) |
458
+ | **Vision Optional** | βœ… (Text Map + OCR) | ❌ (Screenshots only) | ❌ (Screenshots only) | ❌ (Accessibility only) | ❌ (Accessibility only) |
459
+ | **Smart Click-by-Name**| βœ… (Fuzzy name matching) | ⚠️ (Element Index only) | ❌ (Coordinates only) | ❌ (Coordinates only) | ⚠️ (COM API target) |
460
+ | **UI Polling & Wait** | βœ… (`wait_for_element`) | ❌ (Manual sleeps) | ❌ (Manual sleeps) | ❌ (Manual sleeps) | ❌ (Manual sleeps) |
461
+ | **Human Mimicry** | βœ… (BΓ©zier curves + typing) | ❌ (Instant click/type) | ❌ (Instant click/type) | ❌ (Instant click/type) | ❌ (Instant click/type) |
462
+ | **Zero VM Required** | βœ… (`pip install -e .`) | ❌ (Requires Lume/VM) | βœ… (`pip install`) | βœ… (`pip install`) | ⚠️ (Requires setup) |
463
+ | **Clipboard Support** | βœ… (Native Win32 & ctypes) | βœ… (Command wrapper) | ❌ | ❌ | ❌ |
464
+ | **Multi-Monitor** | βœ… (Native resolution maps) | ⚠️ (Primary display) | ❌ | ⚠️ (Primary display) | ❌ |
465
+ | **Safety Mode** | βœ… (Regex action patterns) | ⚠️ (VM-dependent) | ❌ | ❌ | ❌ |
466
+
467
+ ---
468
+
469
+ ## πŸ—οΈ Architecture
470
+
471
+ <p align="center">
472
+ <img src="assets/architecture_diagram.png" alt="UACC Architecture β€” Observe β†’ Think β†’ Act β†’ Verify" width="700">
473
+ </p>
474
+
475
+ ### The Agent Loop
476
+
477
+ ```
478
+ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
479
+ β”‚ UACC Agent Loop β”‚
480
+ β”‚ β”‚
481
+ β”‚ πŸ‘οΈ OBSERVE ──→ 🧠 THINK ──→ ▢️ ACT ──→ βœ… VERIFY β”‚
482
+ β”‚ ↑ β”‚ β”‚
483
+ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚
484
+ β”‚ β”‚
485
+ β”‚ 1. Capture screen + build text map/grid overlay β”‚
486
+ β”‚ 2. Send to LLM: "What action should I take?" β”‚
487
+ β”‚ 3. Execute action with human-like mouse/keyboard β”‚
488
+ β”‚ 4. Verify the action had the expected effect β”‚
489
+ β”‚ 5. Repeat until task is complete β”‚
490
+ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
491
+ ```
492
+
493
+ ### System Architecture
494
+
495
+ ```
496
+ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
497
+ β”‚ AI Agent / MCP Client β”‚
498
+ β”‚ Claude Code β”‚ Hermes β”‚ OpenCode β”‚ OpenClaw β”‚
499
+ β”‚ Claude Desktop β”‚ Cursor β”‚ Any MCP Client β”‚
500
+ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
501
+ β”‚ MCP Protocol β”‚
502
+ β”‚ stdio β”‚ SSE β”‚ Streamable HTTP β”‚
503
+ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
504
+ β”‚ UACC MCP Server β”‚
505
+ β”‚ screenshot β”‚ click β”‚ type β”‚ hotkey β”‚ scroll β”‚
506
+ β”‚ drag β”‚ hover β”‚ find_element β”‚ get_screen_info β”‚
507
+ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
508
+ β”‚ LLM Adapters β”‚
509
+ β”‚ Text β”‚ Vision β”‚ Hybrid β”‚
510
+ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
511
+ β”‚ UACC Core Engine β”‚
512
+ β”‚ Text Map β”‚ Grid Encoder β”‚ Screen Diff β”‚
513
+ β”‚ Accessibility Tree β”‚ OCR β”‚ Safe Mode β”‚
514
+ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
515
+ β”‚ Platform Layer β”‚
516
+ β”‚ pyautogui β”‚ pywinauto β”‚ EasyOCR β”‚ mss β”‚
517
+ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
518
+ ```
519
+
520
+ ---
521
+
522
+ ## πŸ“‚ Project Structure
523
+
524
+ ```
525
+ uacc/
526
+ β”œβ”€β”€ uacc/
527
+ β”‚ β”œβ”€β”€ config.py # Central configuration
528
+ β”‚ β”œβ”€β”€ core/
529
+ β”‚ β”‚ β”œβ”€β”€ screen_capture.py # Fast screenshot capture (mss)
530
+ β”‚ β”‚ β”œβ”€β”€ accessibility.py # Windows UI Automation tree
531
+ β”‚ β”‚ β”œβ”€β”€ ocr_engine.py # Text extraction (EasyOCR)
532
+ β”‚ β”‚ β”œβ”€β”€ text_map.py # Structured screen representation
533
+ β”‚ β”‚ β”œβ”€β”€ grid_encoder.py # Coordinate grid + markers
534
+ β”‚ β”‚ └── screen_diff.py # Screen change detection
535
+ β”‚ β”œβ”€β”€ actions/
536
+ β”‚ β”‚ β”œβ”€β”€ schema.py # Action types (click, drag, type, etc.)
537
+ β”‚ β”‚ β”œβ”€β”€ executor.py # Real input execution (pyautogui)
538
+ β”‚ β”‚ └── human_mimicry.py # BΓ©zier mouse paths, natural typing
539
+ β”‚ β”œβ”€β”€ models/
540
+ β”‚ β”‚ β”œβ”€β”€ base_adapter.py # Abstract LLM interface
541
+ β”‚ β”‚ β”œβ”€β”€ text_adapter.py # For text-only models
542
+ β”‚ β”‚ β”œβ”€β”€ vision_adapter.py # For vision models
543
+ β”‚ β”‚ └── hybrid_adapter.py # Combined text + vision
544
+ β”‚ └── agent/
545
+ β”‚ β”œβ”€β”€ controller.py # Observe β†’ Think β†’ Act β†’ Verify loop
546
+ β”‚ β”œβ”€β”€ memory.py # Session memory & element cache
547
+ β”‚ └── verifier.py # Pre/post action verification
548
+ β”œβ”€β”€ uacc_mcp/ # MCP Server
549
+ β”‚ β”œβ”€β”€ server.py # MCP tools + resources
550
+ β”‚ └── utils.py # Image encoding, session state
551
+ β”œβ”€β”€ examples/ # Demo scripts
552
+ β”œβ”€β”€ tests/ # Unit tests
553
+ β”œβ”€β”€ pyproject.toml
554
+ └── requirements.txt
555
+ ```
556
+
557
+ ---
558
+
559
+ ## πŸ“‹ Examples
560
+
561
+ ```bash
562
+ # See what a text-only LLM "sees" (safe, no mouse control)
563
+ python examples/demo_text_map.py
564
+
565
+ # See what a vision LLM sees (saves grid overlay images)
566
+ python examples/demo_grid.py
567
+
568
+ # Full agent: open Notepad and type text
569
+ python examples/open_notepad.py --mode hybrid
570
+
571
+ # Full agent: search the web
572
+ python examples/web_search.py "What is UACC?"
573
+
574
+ # Creative: draw in MS Paint
575
+ python examples/draw_in_paint.py
576
+ ```
577
+
578
+ ---
579
+
580
+ ## πŸ§ͺ Testing
581
+
582
+ ```bash
583
+ pip install -e ".[dev]"
584
+ python -m pytest tests/ -v
585
+ ```
586
+
587
+ ---
588
+
589
+ ## βš™οΈ Configuration
590
+
591
+ All settings via environment variables or `.env` file:
592
+
593
+ ```bash
594
+ # LLM
595
+ OPENAI_API_KEY=sk-... # Your API key
596
+ OPENAI_MODEL=gpt-4o # Model name
597
+ OPENAI_BASE_URL= # Custom endpoint (Ollama, vLLM, etc.)
598
+
599
+ # UACC
600
+ UACC_MODE=hybrid # text | vision | hybrid
601
+ UACC_SAFE_MODE=true # Block destructive actions
602
+ UACC_HUMAN_MIMICRY=true # BΓ©zier curve mouse movement
603
+ UACC_MAX_ITERATIONS=30 # Max agent iterations per task
604
+ UACC_GRID_MODE=medium # coarse | medium | fine | micro
605
+ ```
606
+
607
+ See [`.env.example`](.env.example) for the full list.
608
+
609
+ ---
610
+
611
+ ## ⚠️ Safety
612
+
613
+ UACC controls your **real mouse and keyboard**. Built-in protections:
614
+
615
+ - πŸ›‘οΈ **Safe Mode** (on by default) β€” blocks destructive patterns (`delete`, `rm -rf`, `format`, etc.)
616
+ - πŸ–±οΈ **pyautogui failsafe** β€” move your mouse to any screen corner to instantly abort
617
+ - πŸ“ **Action logging** β€” every action is logged with coordinates and reasoning
618
+ - πŸ”’ **MCP boundaries** β€” the MCP server only exposes UI tools, no file/shell/network access
619
+
620
+ > **Tip:** Start with `demo_text_map.py` and `demo_grid.py` β€” they're read-only and don't control the mouse.
621
+
622
+ ---
623
+
624
+ ## πŸ—ΊοΈ Roadmap
625
+
626
+ - [ ] 🐧 Linux support (X11/Wayland accessibility)
627
+ - [ ] 🍎 macOS support (AppleScript + Accessibility API)
628
+ - [ ] 🌐 Browser-specific adapter (Playwright/Selenium integration)
629
+ - [ ] πŸ“Έ Video recording of agent sessions
630
+ - [ ] πŸ”„ Multi-monitor support
631
+ - [ ] πŸ“Š Action replay and debugging UI
632
+ - [ ] 🧩 Plugin system for custom actions
633
+ - [ ] πŸ‹οΈ Benchmark suite for agent evaluation
634
+
635
+ ---
636
+
637
+ ## 🀝 Contributing
638
+
639
+ We welcome contributions of all kinds! See our [Contributing Guide](CONTRIBUTING.md) for details.
640
+
641
+ - πŸ› [Report a bug](../../issues/new?template=bug_report.md)
642
+ - ✨ [Request a feature](../../issues/new?template=feature_request.md)
643
+ - πŸ“ Improve documentation
644
+ - πŸ§ͺ Add test coverage
645
+
646
+ **Good first issues** are labeled and waiting for you β€” check the [issues page](../../issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22).
647
+
648
+ ---
649
+
650
+ ## πŸ“„ License
651
+
652
+ [MIT](https://github.com/uacc-project/uacc/blob/main/LICENSE) β€” use it, fork it, ship it.
653
+
654
+ ---
655
+
656
+ ## ⭐ Star History
657
+
658
+ If UACC is useful to you, consider giving it a star! It helps others discover the project.
659
+
660
+ [![Star History Chart](https://api.star-history.com/svg?repos=uacc-project/uacc&type=Date)](https://star-history.com/#uacc-project/uacc&Date)
661
+
662
+ ---
663
+
664
+ <p align="center">
665
+ <strong>Built for the agentic era.</strong><br>
666
+ <em>UACC β€” because AI should work with any model, on any desktop, under your control.</em>
667
+ </p>