computer-control-mcp 0.3.1__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/PKG-INFO +32 -9
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/README.md +31 -8
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/core.py +268 -34
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/pyproject.toml +1 -1
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/.gitignore +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/LICENSE +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/FZYTK.TTF +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/__init__.py +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/__main__.py +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/cli.py +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/gui.py +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/server.py +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/test.py +0 -0
- {computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/test_image.png +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: computer-control-mcp
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies.
|
|
5
5
|
Project-URL: Homepage, https://github.com/AB498/computer-control-mcp
|
|
6
6
|
Project-URL: Issues, https://github.com/AB498/computer-control-mcp/issues
|
|
@@ -33,13 +33,15 @@ Description-Content-Type: text/markdown
|
|
|
33
33
|
|
|
34
34
|
### MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies.
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
36
|
+
<div align="center" style="text-align:center;font-family: monospace; display: flex; align-items: center; justify-content: center; width: 100%; gap: 10px">
|
|
37
|
+
<a href="https://nextjs-boilerplate-ashy-nine-64.vercel.app/demo-computer-control"><img
|
|
38
|
+
src="https://komarev.com/ghpvc/?username=AB498&label=DEMO&style=for-the-badge&color=CC0000" /></a>
|
|
39
|
+
<a href="https://discord.gg/ZeeqSBpjU2"><img
|
|
40
|
+
src="https://img.shields.io/discord/1095854826786668545?style=for-the-badge&color=0000CC" alt="Discord"></a>
|
|
41
|
+
<a href="https://img.shields.io/badge/License-MIT-yellow.svg"><img
|
|
42
|
+
src="https://img.shields.io/badge/License-MIT-yellow.svg?style=for-the-badge&color=00CC00" alt="License: MIT"></a>
|
|
43
|
+
<a href="https://pypi.org/project/computer-control-mcp"><img
|
|
44
|
+
src="https://img.shields.io/pypi/v/computer-control-mcp?style=for-the-badge" alt="PyPi"></a>
|
|
43
45
|
</div>
|
|
44
46
|
|
|
45
47
|
---
|
|
@@ -86,13 +88,19 @@ computer-control-mcp # instead of uvx computer-control-mcp, so you can use the l
|
|
|
86
88
|
- `click_screen(x: int, y: int)`: Click at specified screen coordinates
|
|
87
89
|
- `move_mouse(x: int, y: int)`: Move mouse cursor to specified coordinates
|
|
88
90
|
- `drag_mouse(from_x: int, from_y: int, to_x: int, to_y: int, duration: float = 0.5)`: Drag mouse from one position to another
|
|
91
|
+
- `mouse_down(button: str = "left")`: Hold down a mouse button ('left', 'right', 'middle')
|
|
92
|
+
- `mouse_up(button: str = "left")`: Release a mouse button ('left', 'right', 'middle')
|
|
89
93
|
|
|
90
94
|
### Keyboard Control
|
|
91
95
|
- `type_text(text: str)`: Type the specified text at current cursor position
|
|
92
96
|
- `press_key(key: str)`: Press a specified keyboard key
|
|
97
|
+
- `key_down(key: str)`: Hold down a specific keyboard key until released
|
|
98
|
+
- `key_up(key: str)`: Release a specific keyboard key
|
|
99
|
+
- `press_keys(keys: Union[str, List[Union[str, List[str]]]])`: Press keyboard keys (supports single keys, sequences, and combinations)
|
|
93
100
|
|
|
94
101
|
### Screen and Window Management
|
|
95
|
-
- `take_screenshot(title_pattern: str = None, use_regex: bool = False, threshold: int = 60,
|
|
102
|
+
- `take_screenshot(title_pattern: str = None, use_regex: bool = False, threshold: int = 60, scale_percent_for_ocr: int = None, save_to_downloads: bool = False)`: Capture screen or window
|
|
103
|
+
- `take_screenshot_with_ocr(title_pattern: str = None, use_regex: bool = False, threshold: int = 10, scale_percent_for_ocr: int = None, save_to_downloads: bool = False)`: Extract adn return text with coordinates using OCR from screen or window
|
|
96
104
|
- `get_screen_size()`: Get current screen resolution
|
|
97
105
|
- `list_windows()`: List all open windows
|
|
98
106
|
- `activate_window(title_pattern: str, use_regex: bool = False, threshold: int = 60)`: Bring specified window to foreground
|
|
@@ -111,6 +119,21 @@ pip install -e .
|
|
|
111
119
|
|
|
112
120
|
# Start server
|
|
113
121
|
python -m computer_control_mcp.core
|
|
122
|
+
|
|
123
|
+
# -- OR --
|
|
124
|
+
|
|
125
|
+
# Build
|
|
126
|
+
hatch build
|
|
127
|
+
|
|
128
|
+
# Non-windows
|
|
129
|
+
pip install dist/*.whl --upgrade
|
|
130
|
+
|
|
131
|
+
# Windows
|
|
132
|
+
$latest = Get-ChildItem .\dist\*.whl | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
|
133
|
+
pip install $latest.FullName --upgrade
|
|
134
|
+
|
|
135
|
+
# Run
|
|
136
|
+
computer-control-mcp
|
|
114
137
|
```
|
|
115
138
|
|
|
116
139
|
### Running Tests
|
|
@@ -2,13 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
### MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
5
|
+
<div align="center" style="text-align:center;font-family: monospace; display: flex; align-items: center; justify-content: center; width: 100%; gap: 10px">
|
|
6
|
+
<a href="https://nextjs-boilerplate-ashy-nine-64.vercel.app/demo-computer-control"><img
|
|
7
|
+
src="https://komarev.com/ghpvc/?username=AB498&label=DEMO&style=for-the-badge&color=CC0000" /></a>
|
|
8
|
+
<a href="https://discord.gg/ZeeqSBpjU2"><img
|
|
9
|
+
src="https://img.shields.io/discord/1095854826786668545?style=for-the-badge&color=0000CC" alt="Discord"></a>
|
|
10
|
+
<a href="https://img.shields.io/badge/License-MIT-yellow.svg"><img
|
|
11
|
+
src="https://img.shields.io/badge/License-MIT-yellow.svg?style=for-the-badge&color=00CC00" alt="License: MIT"></a>
|
|
12
|
+
<a href="https://pypi.org/project/computer-control-mcp"><img
|
|
13
|
+
src="https://img.shields.io/pypi/v/computer-control-mcp?style=for-the-badge" alt="PyPi"></a>
|
|
12
14
|
</div>
|
|
13
15
|
|
|
14
16
|
---
|
|
@@ -55,13 +57,19 @@ computer-control-mcp # instead of uvx computer-control-mcp, so you can use the l
|
|
|
55
57
|
- `click_screen(x: int, y: int)`: Click at specified screen coordinates
|
|
56
58
|
- `move_mouse(x: int, y: int)`: Move mouse cursor to specified coordinates
|
|
57
59
|
- `drag_mouse(from_x: int, from_y: int, to_x: int, to_y: int, duration: float = 0.5)`: Drag mouse from one position to another
|
|
60
|
+
- `mouse_down(button: str = "left")`: Hold down a mouse button ('left', 'right', 'middle')
|
|
61
|
+
- `mouse_up(button: str = "left")`: Release a mouse button ('left', 'right', 'middle')
|
|
58
62
|
|
|
59
63
|
### Keyboard Control
|
|
60
64
|
- `type_text(text: str)`: Type the specified text at current cursor position
|
|
61
65
|
- `press_key(key: str)`: Press a specified keyboard key
|
|
66
|
+
- `key_down(key: str)`: Hold down a specific keyboard key until released
|
|
67
|
+
- `key_up(key: str)`: Release a specific keyboard key
|
|
68
|
+
- `press_keys(keys: Union[str, List[Union[str, List[str]]]])`: Press keyboard keys (supports single keys, sequences, and combinations)
|
|
62
69
|
|
|
63
70
|
### Screen and Window Management
|
|
64
|
-
- `take_screenshot(title_pattern: str = None, use_regex: bool = False, threshold: int = 60,
|
|
71
|
+
- `take_screenshot(title_pattern: str = None, use_regex: bool = False, threshold: int = 60, scale_percent_for_ocr: int = None, save_to_downloads: bool = False)`: Capture screen or window
|
|
72
|
+
- `take_screenshot_with_ocr(title_pattern: str = None, use_regex: bool = False, threshold: int = 10, scale_percent_for_ocr: int = None, save_to_downloads: bool = False)`: Extract adn return text with coordinates using OCR from screen or window
|
|
65
73
|
- `get_screen_size()`: Get current screen resolution
|
|
66
74
|
- `list_windows()`: List all open windows
|
|
67
75
|
- `activate_window(title_pattern: str, use_regex: bool = False, threshold: int = 60)`: Bring specified window to foreground
|
|
@@ -80,6 +88,21 @@ pip install -e .
|
|
|
80
88
|
|
|
81
89
|
# Start server
|
|
82
90
|
python -m computer_control_mcp.core
|
|
91
|
+
|
|
92
|
+
# -- OR --
|
|
93
|
+
|
|
94
|
+
# Build
|
|
95
|
+
hatch build
|
|
96
|
+
|
|
97
|
+
# Non-windows
|
|
98
|
+
pip install dist/*.whl --upgrade
|
|
99
|
+
|
|
100
|
+
# Windows
|
|
101
|
+
$latest = Get-ChildItem .\dist\*.whl | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
|
102
|
+
pip install $latest.FullName --upgrade
|
|
103
|
+
|
|
104
|
+
# Run
|
|
105
|
+
computer-control-mcp
|
|
83
106
|
```
|
|
84
107
|
|
|
85
108
|
### Running Tests
|
|
@@ -17,16 +17,18 @@ import uuid
|
|
|
17
17
|
import datetime
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
import tempfile
|
|
20
|
+
from typing import Union
|
|
20
21
|
|
|
21
22
|
# --- Auto-install dependencies if needed ---
|
|
22
23
|
import pyautogui
|
|
23
24
|
from mcp.server.fastmcp import FastMCP, Image
|
|
24
25
|
import mss
|
|
25
26
|
from PIL import Image as PILImage
|
|
27
|
+
|
|
26
28
|
try:
|
|
27
|
-
import pygetwindow as gw
|
|
28
|
-
except (NotImplementedError, ImportError):
|
|
29
29
|
import pywinctl as gw
|
|
30
|
+
except (NotImplementedError, ImportError):
|
|
31
|
+
import pygetwindow as gw
|
|
30
32
|
from fuzzywuzzy import fuzz, process
|
|
31
33
|
|
|
32
34
|
import cv2
|
|
@@ -34,7 +36,7 @@ from rapidocr import RapidOCR
|
|
|
34
36
|
|
|
35
37
|
from pydantic import BaseModel
|
|
36
38
|
|
|
37
|
-
BaseModel.model_config = {
|
|
39
|
+
BaseModel.model_config = {"arbitrary_types_allowed": True}
|
|
38
40
|
|
|
39
41
|
engine = RapidOCR()
|
|
40
42
|
|
|
@@ -46,9 +48,19 @@ RELOAD_ENABLED = True # Set to False to disable auto-reload
|
|
|
46
48
|
mcp = FastMCP("ComputerControlMCP")
|
|
47
49
|
|
|
48
50
|
|
|
51
|
+
# Determine mode automatically
|
|
52
|
+
IS_DEVELOPMENT = os.getenv("ENV") == "development"
|
|
53
|
+
|
|
54
|
+
|
|
49
55
|
def log(message: str) -> None:
|
|
50
|
-
"""Log
|
|
51
|
-
|
|
56
|
+
"""Log to stderr in dev, to stdout or file in production."""
|
|
57
|
+
if IS_DEVELOPMENT:
|
|
58
|
+
# In dev, write to stderr
|
|
59
|
+
print(f"[DEV] {message}", file=sys.stderr)
|
|
60
|
+
else:
|
|
61
|
+
# In production, write to stdout or a file
|
|
62
|
+
print(f"[PROD] {message}", file=sys.stdout)
|
|
63
|
+
# or append to a file: open("app.log", "a").write(message+"\n")
|
|
52
64
|
|
|
53
65
|
|
|
54
66
|
def get_downloads_dir() -> Path:
|
|
@@ -67,10 +79,10 @@ def get_downloads_dir() -> Path:
|
|
|
67
79
|
|
|
68
80
|
def _mss_screenshot(region=None):
|
|
69
81
|
"""Take a screenshot using mss and return PIL Image.
|
|
70
|
-
|
|
82
|
+
|
|
71
83
|
Args:
|
|
72
84
|
region: Optional tuple (left, top, width, height) for region capture
|
|
73
|
-
|
|
85
|
+
|
|
74
86
|
Returns:
|
|
75
87
|
PIL Image object
|
|
76
88
|
"""
|
|
@@ -87,10 +99,12 @@ def _mss_screenshot(region=None):
|
|
|
87
99
|
"width": width,
|
|
88
100
|
"height": height,
|
|
89
101
|
}
|
|
90
|
-
|
|
102
|
+
|
|
91
103
|
screenshot = sct.grab(monitor)
|
|
92
104
|
# Convert to PIL Image
|
|
93
|
-
return PILImage.frombytes(
|
|
105
|
+
return PILImage.frombytes(
|
|
106
|
+
"RGB", screenshot.size, screenshot.bgra, "raw", "BGRX"
|
|
107
|
+
)
|
|
94
108
|
|
|
95
109
|
|
|
96
110
|
def save_image_to_downloads(
|
|
@@ -137,7 +151,7 @@ def _find_matching_window(
|
|
|
137
151
|
windows: any,
|
|
138
152
|
title_pattern: str = None,
|
|
139
153
|
use_regex: bool = False,
|
|
140
|
-
threshold: int =
|
|
154
|
+
threshold: int = 10,
|
|
141
155
|
) -> Optional[Dict[str, Any]]:
|
|
142
156
|
"""Helper function to find a matching window based on title pattern.
|
|
143
157
|
|
|
@@ -184,6 +198,7 @@ def _find_matching_window(
|
|
|
184
198
|
|
|
185
199
|
# --- MCP Function Handlers ---
|
|
186
200
|
|
|
201
|
+
|
|
187
202
|
@mcp.tool()
|
|
188
203
|
def click_screen(x: int, y: int) -> str:
|
|
189
204
|
"""Click at the specified screen coordinates."""
|
|
@@ -222,28 +237,165 @@ def type_text(text: str) -> str:
|
|
|
222
237
|
def take_screenshot(
|
|
223
238
|
title_pattern: str = None,
|
|
224
239
|
use_regex: bool = False,
|
|
225
|
-
threshold: int =
|
|
226
|
-
with_ocr_text_and_coords: bool = False,
|
|
240
|
+
threshold: int = 10,
|
|
227
241
|
scale_percent_for_ocr: int = None,
|
|
228
242
|
save_to_downloads: bool = False,
|
|
229
|
-
) -> Image
|
|
243
|
+
) -> Image:
|
|
230
244
|
"""
|
|
231
|
-
Get screenshot
|
|
232
|
-
If no title pattern is provided, get screenshot of entire screen and all text on the screen.
|
|
245
|
+
Get screenshot Image as MCP Image object. If no title pattern is provided, get screenshot of entire screen and all text on the screen.
|
|
233
246
|
|
|
234
247
|
Args:
|
|
235
248
|
title_pattern: Pattern to match window title, if None, take screenshot of entire screen
|
|
236
249
|
use_regex: If True, treat the pattern as a regex, otherwise best match with fuzzy matching
|
|
237
250
|
threshold: Minimum score (0-100) required for a fuzzy match
|
|
238
|
-
with_ocr_text_and_coords: If True, get OCR text with absolute coordinates from the screenshot
|
|
239
251
|
scale_percent_for_ocr: Percentage to scale the image down before processing, you wont need this most of the time unless your pc is extremely old or slow
|
|
240
252
|
save_to_downloads: If True, save the screenshot to the downloads directory and return the absolute path
|
|
241
253
|
|
|
242
254
|
Returns:
|
|
243
|
-
Returns a single screenshot as MCP Image object
|
|
255
|
+
Returns a single screenshot as MCP Image object. "content type image not supported" means preview isnt supported but Image object is there and returned successfully.
|
|
244
256
|
"""
|
|
245
257
|
try:
|
|
258
|
+
all_windows = gw.getAllWindows()
|
|
259
|
+
|
|
260
|
+
# Convert to list of dictionaries for _find_matching_window
|
|
261
|
+
windows = []
|
|
262
|
+
for window in all_windows:
|
|
263
|
+
if window.title: # Only include windows with titles
|
|
264
|
+
windows.append(
|
|
265
|
+
{
|
|
266
|
+
"title": window.title,
|
|
267
|
+
"window_obj": window, # Store the actual window object
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
log(f"Found {len(windows)} windows")
|
|
272
|
+
window = _find_matching_window(windows, title_pattern, use_regex, threshold)
|
|
273
|
+
window = window["window_obj"] if window else None
|
|
274
|
+
|
|
275
|
+
import ctypes
|
|
276
|
+
import time
|
|
277
|
+
|
|
278
|
+
def force_activate(window):
|
|
279
|
+
"""Force a window to the foreground on Windows."""
|
|
280
|
+
try:
|
|
281
|
+
hwnd = window._hWnd # pywinctl window handle
|
|
282
|
+
|
|
283
|
+
# Restore if minimized
|
|
284
|
+
if window.isMinimized:
|
|
285
|
+
window.restore()
|
|
286
|
+
time.sleep(0.1)
|
|
287
|
+
|
|
288
|
+
# Bring to top and set foreground
|
|
289
|
+
ctypes.windll.user32.SetForegroundWindow(hwnd)
|
|
290
|
+
ctypes.windll.user32.BringWindowToTop(hwnd)
|
|
291
|
+
window.activate() # fallback
|
|
292
|
+
time.sleep(0.3) # wait for OS to update
|
|
246
293
|
|
|
294
|
+
except Exception as e:
|
|
295
|
+
print(f"Warning: Could not force window: {e}", file=sys.stderr)
|
|
296
|
+
|
|
297
|
+
# Take the screenshot
|
|
298
|
+
if not window:
|
|
299
|
+
log("No matching window found, taking screenshot of entire screen")
|
|
300
|
+
screenshot = _mss_screenshot()
|
|
301
|
+
else:
|
|
302
|
+
try:
|
|
303
|
+
# Re-fetch window handle to ensure it's valid
|
|
304
|
+
window = gw.getWindowsWithTitle(window.title)[0]
|
|
305
|
+
current_active_window = gw.getActiveWindow()
|
|
306
|
+
log(f"Taking screenshot of window: {window.title}")
|
|
307
|
+
|
|
308
|
+
if sys.platform == "win32":
|
|
309
|
+
force_activate(window)
|
|
310
|
+
else:
|
|
311
|
+
window.activate()
|
|
312
|
+
pyautogui.sleep(0.5) # Give Windows time to focus
|
|
313
|
+
|
|
314
|
+
screen_width, screen_height = pyautogui.size()
|
|
315
|
+
|
|
316
|
+
screenshot = _mss_screenshot(
|
|
317
|
+
region=(
|
|
318
|
+
max(window.left, 0),
|
|
319
|
+
max(window.top, 0),
|
|
320
|
+
min(window.width, screen_width),
|
|
321
|
+
min(window.height, screen_height),
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
# Restore previously active window
|
|
326
|
+
if current_active_window and current_active_window != window:
|
|
327
|
+
try:
|
|
328
|
+
if sys.platform == "win32":
|
|
329
|
+
force_activate(current_active_window)
|
|
330
|
+
else:
|
|
331
|
+
current_active_window.activate()
|
|
332
|
+
pyautogui.sleep(0.2)
|
|
333
|
+
except Exception as e:
|
|
334
|
+
log(f"Error restoring previous window: {str(e)}")
|
|
335
|
+
except Exception as e:
|
|
336
|
+
log(f"Error taking screenshot of window: {str(e)}")
|
|
337
|
+
screenshot = _mss_screenshot() # fallback to full screen
|
|
338
|
+
|
|
339
|
+
# Create temp directory
|
|
340
|
+
temp_dir = Path(tempfile.mkdtemp())
|
|
341
|
+
|
|
342
|
+
# Save screenshot and get filepath
|
|
343
|
+
filepath, _ = save_image_to_downloads(
|
|
344
|
+
screenshot, prefix="screenshot", directory=temp_dir
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
# Create Image object from filepath
|
|
348
|
+
image = Image(filepath)
|
|
349
|
+
|
|
350
|
+
if save_to_downloads:
|
|
351
|
+
log("Copying screenshot from temp to downloads")
|
|
352
|
+
shutil.copy(filepath, get_downloads_dir())
|
|
353
|
+
|
|
354
|
+
return image # MCP Image object
|
|
355
|
+
|
|
356
|
+
except Exception as e:
|
|
357
|
+
log(f"Error in screenshot or getting UI elements: {str(e)}")
|
|
358
|
+
import traceback
|
|
359
|
+
|
|
360
|
+
stack_trace = traceback.format_exc()
|
|
361
|
+
log(f"Stack trace:\n{stack_trace}")
|
|
362
|
+
return f"Error in screenshot or getting UI elements: {str(e)}\nStack trace:\n{stack_trace}"
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def is_low_spec_pc() -> bool:
|
|
366
|
+
try:
|
|
367
|
+
import psutil
|
|
368
|
+
|
|
369
|
+
cpu_low = psutil.cpu_count(logical=False) < 4
|
|
370
|
+
ram_low = psutil.virtual_memory().total < 8 * 1024**3
|
|
371
|
+
return cpu_low or ram_low
|
|
372
|
+
except Exception:
|
|
373
|
+
# Fallback if psutil not available or info unavailable
|
|
374
|
+
return False
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@mcp.tool()
|
|
378
|
+
def take_screenshot_with_ocr(
|
|
379
|
+
title_pattern: str = None,
|
|
380
|
+
use_regex: bool = False,
|
|
381
|
+
threshold: int = 10,
|
|
382
|
+
scale_percent_for_ocr: int = None,
|
|
383
|
+
save_to_downloads: bool = False,
|
|
384
|
+
) -> str:
|
|
385
|
+
"""
|
|
386
|
+
Get OCR text from screenshot with absolute coordinates as JSON string of List[Tuple[List[List[int]], str, float]] (returned after adding the window offset from true (0, 0) of screen to the OCR coordinates, so clicking is on-point. Recommended to click in the middle of OCR Box) and using confidence from window with the specified title pattern. If no title pattern is provided, get screenshot of entire screen and all text on the screen. Know that OCR takes around 20 seconds on an mid-spec pc at 1080p resolution.
|
|
387
|
+
|
|
388
|
+
Args:
|
|
389
|
+
title_pattern: Pattern to match window title, if None, take screenshot of entire screen
|
|
390
|
+
use_regex: If True, treat the pattern as a regex, otherwise best match with fuzzy matching
|
|
391
|
+
threshold: Minimum score (0-100) required for a fuzzy match
|
|
392
|
+
scale_percent_for_ocr: Percentage to scale the image down before processing, you wont need this most of the time unless your pc is extremely old or slow
|
|
393
|
+
save_to_downloads: If True, save the screenshot to the downloads directory and return the absolute path
|
|
394
|
+
|
|
395
|
+
Returns:
|
|
396
|
+
Returns a list of UI elements as List[Tuple[List[List[int]], str, float]] where each tuple is [[4 corners of box], text, confidence], "content type image not supported" means preview isnt supported but Image object is there.
|
|
397
|
+
"""
|
|
398
|
+
try:
|
|
247
399
|
all_windows = gw.getAllWindows()
|
|
248
400
|
|
|
249
401
|
# Convert to list of dictionaries for _find_matching_window
|
|
@@ -272,7 +424,6 @@ def take_screenshot(
|
|
|
272
424
|
log(f"Taking screenshot of window: {window.title}")
|
|
273
425
|
# Activate the window and wait for it to be fully in focus
|
|
274
426
|
try:
|
|
275
|
-
|
|
276
427
|
window.activate()
|
|
277
428
|
pyautogui.sleep(0.5) # Wait for 0.5 seconds to ensure window is active
|
|
278
429
|
screenshot = _mss_screenshot(
|
|
@@ -282,7 +433,9 @@ def take_screenshot(
|
|
|
282
433
|
if current_active_window:
|
|
283
434
|
try:
|
|
284
435
|
current_active_window.activate()
|
|
285
|
-
pyautogui.sleep(
|
|
436
|
+
pyautogui.sleep(
|
|
437
|
+
0.2
|
|
438
|
+
) # Wait a bit to ensure previous window is restored
|
|
286
439
|
except Exception as e:
|
|
287
440
|
log(f"Error restoring previous window: {str(e)}")
|
|
288
441
|
except Exception as e:
|
|
@@ -300,9 +453,6 @@ def take_screenshot(
|
|
|
300
453
|
# Create Image object from filepath
|
|
301
454
|
image = Image(filepath)
|
|
302
455
|
|
|
303
|
-
if not with_ocr_text_and_coords:
|
|
304
|
-
return image # MCP Image object
|
|
305
|
-
|
|
306
456
|
# Copy from temp to downloads
|
|
307
457
|
if save_to_downloads:
|
|
308
458
|
log("Copying screenshot from temp to downloads")
|
|
@@ -313,7 +463,7 @@ def take_screenshot(
|
|
|
313
463
|
|
|
314
464
|
if scale_percent_for_ocr is None:
|
|
315
465
|
# Calculate percent to scale height to 360 pixels
|
|
316
|
-
scale_percent_for_ocr = 100
|
|
466
|
+
scale_percent_for_ocr = 100 # 360 / img.shape[0] * 100
|
|
317
467
|
|
|
318
468
|
# Lower down resolution before processing
|
|
319
469
|
width = int(img.shape[1] * scale_percent_for_ocr / 100)
|
|
@@ -322,20 +472,28 @@ def take_screenshot(
|
|
|
322
472
|
resized_img = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
|
|
323
473
|
# save resized image to pwd
|
|
324
474
|
# cv2.imwrite("resized_img.png", resized_img)
|
|
325
|
-
|
|
475
|
+
|
|
326
476
|
output = engine(resized_img)
|
|
327
|
-
boxes
|
|
328
|
-
txts
|
|
477
|
+
boxes = output.boxes
|
|
478
|
+
txts = output.txts
|
|
329
479
|
scores = output.scores
|
|
330
480
|
zipped_results = list(zip(boxes, txts, scores))
|
|
331
481
|
zipped_results = [
|
|
332
|
-
(
|
|
482
|
+
(
|
|
483
|
+
box.tolist(),
|
|
484
|
+
text,
|
|
485
|
+
float(score),
|
|
486
|
+
) # convert np.array -> list, ensure score is float
|
|
333
487
|
for box, text, score in zipped_results
|
|
334
488
|
]
|
|
335
489
|
log(f"Found {len(zipped_results)} text items in OCR result.")
|
|
336
490
|
log(f"First 5 items: {zipped_results[:5]}")
|
|
337
|
-
return
|
|
338
|
-
|
|
491
|
+
return (
|
|
492
|
+
",\n".join([str(item) for item in zipped_results])
|
|
493
|
+
if zipped_results
|
|
494
|
+
else "No text found"
|
|
495
|
+
)
|
|
496
|
+
|
|
339
497
|
except Exception as e:
|
|
340
498
|
log(f"Error in screenshot or getting UI elements: {str(e)}")
|
|
341
499
|
import traceback
|
|
@@ -355,6 +513,26 @@ def move_mouse(x: int, y: int) -> str:
|
|
|
355
513
|
return f"Error moving mouse to coordinates ({x}, {y}): {str(e)}"
|
|
356
514
|
|
|
357
515
|
|
|
516
|
+
@mcp.tool()
|
|
517
|
+
def mouse_down(button: str = "left") -> str:
|
|
518
|
+
"""Hold down a mouse button ('left', 'right', 'middle')."""
|
|
519
|
+
try:
|
|
520
|
+
pyautogui.mouseDown(button=button)
|
|
521
|
+
return f"Held down {button} mouse button"
|
|
522
|
+
except Exception as e:
|
|
523
|
+
return f"Error holding {button} mouse button: {str(e)}"
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
@mcp.tool()
|
|
527
|
+
def mouse_up(button: str = "left") -> str:
|
|
528
|
+
"""Release a mouse button ('left', 'right', 'middle')."""
|
|
529
|
+
try:
|
|
530
|
+
pyautogui.mouseUp(button=button)
|
|
531
|
+
return f"Released {button} mouse button"
|
|
532
|
+
except Exception as e:
|
|
533
|
+
return f"Error releasing {button} mouse button: {str(e)}"
|
|
534
|
+
|
|
535
|
+
|
|
358
536
|
@mcp.tool()
|
|
359
537
|
async def drag_mouse(
|
|
360
538
|
from_x: int, from_y: int, to_x: int, to_y: int, duration: float = 0.5
|
|
@@ -384,14 +562,69 @@ async def drag_mouse(
|
|
|
384
562
|
return f"Error dragging from ({from_x}, {from_y}) to ({to_x}, {to_y}): {str(e)}"
|
|
385
563
|
|
|
386
564
|
|
|
565
|
+
import pyautogui
|
|
566
|
+
from typing import Union, List
|
|
567
|
+
|
|
568
|
+
|
|
387
569
|
@mcp.tool()
|
|
388
|
-
def
|
|
389
|
-
"""
|
|
570
|
+
def key_down(key: str) -> str:
|
|
571
|
+
"""Hold down a specific keyboard key until released."""
|
|
390
572
|
try:
|
|
391
|
-
pyautogui.
|
|
392
|
-
return f"
|
|
573
|
+
pyautogui.keyDown(key)
|
|
574
|
+
return f"Held down key: {key}"
|
|
575
|
+
except Exception as e:
|
|
576
|
+
return f"Error holding key {key}: {str(e)}"
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
@mcp.tool()
|
|
580
|
+
def key_up(key: str) -> str:
|
|
581
|
+
"""Release a specific keyboard key."""
|
|
582
|
+
try:
|
|
583
|
+
pyautogui.keyUp(key)
|
|
584
|
+
return f"Released key: {key}"
|
|
585
|
+
except Exception as e:
|
|
586
|
+
return f"Error releasing key {key}: {str(e)}"
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
@mcp.tool()
|
|
590
|
+
def press_keys(keys: Union[str, List[Union[str, List[str]]]]) -> str:
|
|
591
|
+
"""
|
|
592
|
+
Press keyboard keys.
|
|
593
|
+
|
|
594
|
+
Args:
|
|
595
|
+
keys:
|
|
596
|
+
- Single key as string (e.g., "enter")
|
|
597
|
+
- Sequence of keys as list (e.g., ["a", "b", "c"])
|
|
598
|
+
- Key combinations as nested list (e.g., [["ctrl", "c"], ["alt", "tab"]])
|
|
599
|
+
|
|
600
|
+
Examples:
|
|
601
|
+
press_keys("enter")
|
|
602
|
+
press_keys(["a", "b", "c"])
|
|
603
|
+
press_keys([["ctrl", "c"], ["alt", "tab"]])
|
|
604
|
+
"""
|
|
605
|
+
try:
|
|
606
|
+
if isinstance(keys, str):
|
|
607
|
+
# Single key
|
|
608
|
+
pyautogui.press(keys)
|
|
609
|
+
return f"Pressed single key: {keys}"
|
|
610
|
+
|
|
611
|
+
elif isinstance(keys, list):
|
|
612
|
+
for item in keys:
|
|
613
|
+
if isinstance(item, str):
|
|
614
|
+
# Sequential key press
|
|
615
|
+
pyautogui.press(item)
|
|
616
|
+
elif isinstance(item, list):
|
|
617
|
+
# Key combination (e.g., ctrl+c)
|
|
618
|
+
pyautogui.hotkey(*item)
|
|
619
|
+
else:
|
|
620
|
+
return f"Invalid key format: {item}"
|
|
621
|
+
return f"Successfully pressed keys sequence: {keys}"
|
|
622
|
+
|
|
623
|
+
else:
|
|
624
|
+
return "Invalid input: must be str or list"
|
|
625
|
+
|
|
393
626
|
except Exception as e:
|
|
394
|
-
return f"Error pressing
|
|
627
|
+
return f"Error pressing keys {keys}: {str(e)}"
|
|
395
628
|
|
|
396
629
|
|
|
397
630
|
@mcp.tool()
|
|
@@ -486,6 +719,7 @@ def main():
|
|
|
486
719
|
|
|
487
720
|
try:
|
|
488
721
|
# Run the server
|
|
722
|
+
log("Computer Control MCP Server Started...")
|
|
489
723
|
mcp.run()
|
|
490
724
|
|
|
491
725
|
except KeyboardInterrupt:
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "computer-control-mcp"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.4"
|
|
8
8
|
description = "MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{computer_control_mcp-0.3.1 → computer_control_mcp-0.3.4}/computer_control_mcp/test_image.png
RENAMED
|
File without changes
|