computer-control-mcp 0.3.0__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/PKG-INFO +14 -9
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/README.md +9 -6
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/core.py +300 -32
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/test.py +34 -4
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/pyproject.toml +50 -48
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/.gitignore +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/LICENSE +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/FZYTK.TTF +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/__init__.py +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/__main__.py +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/cli.py +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/gui.py +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/server.py +0 -0
- {computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/test_image.png +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: computer-control-mcp
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies.
|
|
5
5
|
Project-URL: Homepage, https://github.com/AB498/computer-control-mcp
|
|
6
6
|
Project-URL: Issues, https://github.com/AB498/computer-control-mcp/issues
|
|
@@ -18,12 +18,14 @@ Classifier: Topic :: Utilities
|
|
|
18
18
|
Requires-Python: >=3.10
|
|
19
19
|
Requires-Dist: fuzzywuzzy==0.18.0
|
|
20
20
|
Requires-Dist: mcp[cli]==1.13.0
|
|
21
|
-
Requires-Dist:
|
|
21
|
+
Requires-Dist: mss>=7.0.0
|
|
22
|
+
Requires-Dist: onnxruntime==1.22.0
|
|
22
23
|
Requires-Dist: opencv-python==4.12.0.88
|
|
23
24
|
Requires-Dist: pillow==11.3.0
|
|
24
25
|
Requires-Dist: pyautogui==0.9.54
|
|
25
26
|
Requires-Dist: pygetwindow==0.0.9
|
|
26
|
-
Requires-Dist:
|
|
27
|
+
Requires-Dist: pywinctl==0.4.1
|
|
28
|
+
Requires-Dist: rapidocr-onnxruntime==1.2.3
|
|
27
29
|
Requires-Dist: rapidocr==3.3.1
|
|
28
30
|
Description-Content-Type: text/markdown
|
|
29
31
|
|
|
@@ -31,12 +33,15 @@ Description-Content-Type: text/markdown
|
|
|
31
33
|
|
|
32
34
|
### MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies.
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
36
|
+
<div align="center" style="text-align:center;font-family: monospace; display: flex; align-items: center; justify-content: center; width: 100%; gap: 10px">
|
|
37
|
+
<a href="https://nextjs-boilerplate-ashy-nine-64.vercel.app/demo-computer-control"><img
|
|
38
|
+
src="https://komarev.com/ghpvc/?username=AB498&label=DEMO&style=for-the-badge&color=CC0000" /></a>
|
|
39
|
+
<a href="https://discord.gg/ZeeqSBpjU2"><img
|
|
40
|
+
src="https://img.shields.io/discord/1095854826786668545?style=for-the-badge&color=0000CC" alt="Discord"></a>
|
|
41
|
+
<a href="https://img.shields.io/badge/License-MIT-yellow.svg"><img
|
|
42
|
+
src="https://img.shields.io/badge/License-MIT-yellow.svg?style=for-the-badge&color=00CC00" alt="License: MIT"></a>
|
|
43
|
+
<a href="https://pypi.org/project/computer-control-mcp"><img
|
|
44
|
+
src="https://img.shields.io/pypi/v/computer-control-mcp?style=for-the-badge" alt="PyPi"></a>
|
|
40
45
|
</div>
|
|
41
46
|
|
|
42
47
|
---
|
|
@@ -2,12 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
### MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
<div align="center" style="text-align:center;font-family: monospace; display: flex; align-items: center; justify-content: center; width: 100%; gap: 10px">
|
|
6
|
+
<a href="https://nextjs-boilerplate-ashy-nine-64.vercel.app/demo-computer-control"><img
|
|
7
|
+
src="https://komarev.com/ghpvc/?username=AB498&label=DEMO&style=for-the-badge&color=CC0000" /></a>
|
|
8
|
+
<a href="https://discord.gg/ZeeqSBpjU2"><img
|
|
9
|
+
src="https://img.shields.io/discord/1095854826786668545?style=for-the-badge&color=0000CC" alt="Discord"></a>
|
|
10
|
+
<a href="https://img.shields.io/badge/License-MIT-yellow.svg"><img
|
|
11
|
+
src="https://img.shields.io/badge/License-MIT-yellow.svg?style=for-the-badge&color=00CC00" alt="License: MIT"></a>
|
|
12
|
+
<a href="https://pypi.org/project/computer-control-mcp"><img
|
|
13
|
+
src="https://img.shields.io/pypi/v/computer-control-mcp?style=for-the-badge" alt="PyPi"></a>
|
|
11
14
|
</div>
|
|
12
15
|
|
|
13
16
|
---
|
|
@@ -17,23 +17,28 @@ import uuid
|
|
|
17
17
|
import datetime
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
import tempfile
|
|
20
|
+
from typing import Union
|
|
20
21
|
|
|
21
22
|
# --- Auto-install dependencies if needed ---
|
|
22
23
|
import pyautogui
|
|
23
24
|
from mcp.server.fastmcp import FastMCP, Image
|
|
24
|
-
import
|
|
25
|
+
import mss
|
|
26
|
+
from PIL import Image as PILImage
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import pywinctl as gw
|
|
30
|
+
except (NotImplementedError, ImportError):
|
|
31
|
+
import pygetwindow as gw
|
|
25
32
|
from fuzzywuzzy import fuzz, process
|
|
26
33
|
|
|
27
34
|
import cv2
|
|
28
35
|
from rapidocr import RapidOCR
|
|
29
|
-
from rapidocr_onnxruntime import VisRes
|
|
30
36
|
|
|
31
37
|
from pydantic import BaseModel
|
|
32
38
|
|
|
33
|
-
BaseModel.model_config = {
|
|
39
|
+
BaseModel.model_config = {"arbitrary_types_allowed": True}
|
|
34
40
|
|
|
35
41
|
engine = RapidOCR()
|
|
36
|
-
vis = VisRes()
|
|
37
42
|
|
|
38
43
|
|
|
39
44
|
DEBUG = True # Set to False in production
|
|
@@ -43,9 +48,19 @@ RELOAD_ENABLED = True # Set to False to disable auto-reload
|
|
|
43
48
|
mcp = FastMCP("ComputerControlMCP")
|
|
44
49
|
|
|
45
50
|
|
|
51
|
+
# Determine mode automatically
|
|
52
|
+
IS_DEVELOPMENT = os.getenv("ENV") == "development"
|
|
53
|
+
|
|
54
|
+
|
|
46
55
|
def log(message: str) -> None:
|
|
47
|
-
"""Log
|
|
48
|
-
|
|
56
|
+
"""Log to stderr in dev, to stdout or file in production."""
|
|
57
|
+
if IS_DEVELOPMENT:
|
|
58
|
+
# In dev, write to stderr
|
|
59
|
+
print(f"[DEV] {message}", file=sys.stderr)
|
|
60
|
+
else:
|
|
61
|
+
# In production, write to stdout or a file
|
|
62
|
+
print(f"[PROD] {message}", file=sys.stdout)
|
|
63
|
+
# or append to a file: open("app.log", "a").write(message+"\n")
|
|
49
64
|
|
|
50
65
|
|
|
51
66
|
def get_downloads_dir() -> Path:
|
|
@@ -62,6 +77,36 @@ def get_downloads_dir() -> Path:
|
|
|
62
77
|
return Path.home() / "Downloads"
|
|
63
78
|
|
|
64
79
|
|
|
80
|
+
def _mss_screenshot(region=None):
|
|
81
|
+
"""Take a screenshot using mss and return PIL Image.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
region: Optional tuple (left, top, width, height) for region capture
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
PIL Image object
|
|
88
|
+
"""
|
|
89
|
+
with mss.mss() as sct:
|
|
90
|
+
if region is None:
|
|
91
|
+
# Full screen screenshot
|
|
92
|
+
monitor = sct.monitors[0] # All monitors combined
|
|
93
|
+
else:
|
|
94
|
+
# Region screenshot
|
|
95
|
+
left, top, width, height = region
|
|
96
|
+
monitor = {
|
|
97
|
+
"left": left,
|
|
98
|
+
"top": top,
|
|
99
|
+
"width": width,
|
|
100
|
+
"height": height,
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
screenshot = sct.grab(monitor)
|
|
104
|
+
# Convert to PIL Image
|
|
105
|
+
return PILImage.frombytes(
|
|
106
|
+
"RGB", screenshot.size, screenshot.bgra, "raw", "BGRX"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
65
110
|
def save_image_to_downloads(
|
|
66
111
|
image, prefix: str = "screenshot", directory: Path = None
|
|
67
112
|
) -> Tuple[str, bytes]:
|
|
@@ -106,7 +151,7 @@ def _find_matching_window(
|
|
|
106
151
|
windows: any,
|
|
107
152
|
title_pattern: str = None,
|
|
108
153
|
use_regex: bool = False,
|
|
109
|
-
threshold: int =
|
|
154
|
+
threshold: int = 10,
|
|
110
155
|
) -> Optional[Dict[str, Any]]:
|
|
111
156
|
"""Helper function to find a matching window based on title pattern.
|
|
112
157
|
|
|
@@ -153,6 +198,7 @@ def _find_matching_window(
|
|
|
153
198
|
|
|
154
199
|
# --- MCP Function Handlers ---
|
|
155
200
|
|
|
201
|
+
|
|
156
202
|
@mcp.tool()
|
|
157
203
|
def click_screen(x: int, y: int) -> str:
|
|
158
204
|
"""Click at the specified screen coordinates."""
|
|
@@ -191,28 +237,168 @@ def type_text(text: str) -> str:
|
|
|
191
237
|
def take_screenshot(
|
|
192
238
|
title_pattern: str = None,
|
|
193
239
|
use_regex: bool = False,
|
|
194
|
-
threshold: int =
|
|
195
|
-
with_ocr_text_and_coords: bool = False,
|
|
240
|
+
threshold: int = 10,
|
|
196
241
|
scale_percent_for_ocr: int = None,
|
|
197
242
|
save_to_downloads: bool = False,
|
|
198
|
-
) -> Image
|
|
243
|
+
) -> Image:
|
|
199
244
|
"""
|
|
200
|
-
Get screenshot
|
|
245
|
+
Get screenshot Image as MCP Image object.
|
|
201
246
|
If no title pattern is provided, get screenshot of entire screen and all text on the screen.
|
|
202
247
|
|
|
203
248
|
Args:
|
|
204
249
|
title_pattern: Pattern to match window title, if None, take screenshot of entire screen
|
|
205
250
|
use_regex: If True, treat the pattern as a regex, otherwise best match with fuzzy matching
|
|
206
251
|
threshold: Minimum score (0-100) required for a fuzzy match
|
|
207
|
-
with_ocr_text_and_coords: If True, get OCR text with absolute coordinates from the screenshot
|
|
208
252
|
scale_percent_for_ocr: Percentage to scale the image down before processing, you wont need this most of the time unless your pc is extremely old or slow
|
|
209
253
|
save_to_downloads: If True, save the screenshot to the downloads directory and return the absolute path
|
|
210
254
|
|
|
211
255
|
Returns:
|
|
212
|
-
Returns a single screenshot as MCP Image object
|
|
256
|
+
Returns a single screenshot as MCP Image object. "content type image not supported" means preview isnt supported but Image object is there and returned successfully.
|
|
213
257
|
"""
|
|
214
258
|
try:
|
|
259
|
+
all_windows = gw.getAllWindows()
|
|
260
|
+
|
|
261
|
+
# Convert to list of dictionaries for _find_matching_window
|
|
262
|
+
windows = []
|
|
263
|
+
for window in all_windows:
|
|
264
|
+
if window.title: # Only include windows with titles
|
|
265
|
+
windows.append(
|
|
266
|
+
{
|
|
267
|
+
"title": window.title,
|
|
268
|
+
"window_obj": window, # Store the actual window object
|
|
269
|
+
}
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
log(f"Found {len(windows)} windows")
|
|
273
|
+
window = _find_matching_window(windows, title_pattern, use_regex, threshold)
|
|
274
|
+
window = window["window_obj"] if window else None
|
|
275
|
+
|
|
276
|
+
import ctypes
|
|
277
|
+
import time
|
|
278
|
+
|
|
279
|
+
def force_activate(window):
|
|
280
|
+
"""Force a window to the foreground on Windows."""
|
|
281
|
+
try:
|
|
282
|
+
hwnd = window._hWnd # pywinctl window handle
|
|
283
|
+
|
|
284
|
+
# Restore if minimized
|
|
285
|
+
if window.isMinimized:
|
|
286
|
+
window.restore()
|
|
287
|
+
time.sleep(0.1)
|
|
288
|
+
|
|
289
|
+
# Bring to top and set foreground
|
|
290
|
+
ctypes.windll.user32.SetForegroundWindow(hwnd)
|
|
291
|
+
ctypes.windll.user32.BringWindowToTop(hwnd)
|
|
292
|
+
window.activate() # fallback
|
|
293
|
+
time.sleep(0.3) # wait for OS to update
|
|
294
|
+
|
|
295
|
+
except Exception as e:
|
|
296
|
+
print(f"Warning: Could not force window: {e}", file=sys.stderr)
|
|
297
|
+
|
|
298
|
+
# Take the screenshot
|
|
299
|
+
if not window:
|
|
300
|
+
log("No matching window found, taking screenshot of entire screen")
|
|
301
|
+
screenshot = _mss_screenshot()
|
|
302
|
+
else:
|
|
303
|
+
try:
|
|
304
|
+
# Re-fetch window handle to ensure it's valid
|
|
305
|
+
window = gw.getWindowsWithTitle(window.title)[0]
|
|
306
|
+
current_active_window = gw.getActiveWindow()
|
|
307
|
+
log(f"Taking screenshot of window: {window.title}")
|
|
308
|
+
|
|
309
|
+
if sys.platform == "win32":
|
|
310
|
+
force_activate(window)
|
|
311
|
+
else:
|
|
312
|
+
window.activate()
|
|
313
|
+
pyautogui.sleep(0.5) # Give Windows time to focus
|
|
314
|
+
|
|
315
|
+
screen_width, screen_height = pyautogui.size()
|
|
316
|
+
|
|
317
|
+
screenshot = _mss_screenshot(
|
|
318
|
+
region=(
|
|
319
|
+
max(window.left, 0),
|
|
320
|
+
max(window.top, 0),
|
|
321
|
+
min(window.width, screen_width),
|
|
322
|
+
min(window.height, screen_height),
|
|
323
|
+
)
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
# Restore previously active window
|
|
327
|
+
if current_active_window and current_active_window != window:
|
|
328
|
+
try:
|
|
329
|
+
if sys.platform == "win32":
|
|
330
|
+
force_activate(current_active_window)
|
|
331
|
+
else:
|
|
332
|
+
current_active_window.activate()
|
|
333
|
+
pyautogui.sleep(0.2)
|
|
334
|
+
except Exception as e:
|
|
335
|
+
log(f"Error restoring previous window: {str(e)}")
|
|
336
|
+
except Exception as e:
|
|
337
|
+
log(f"Error taking screenshot of window: {str(e)}")
|
|
338
|
+
screenshot = _mss_screenshot() # fallback to full screen
|
|
339
|
+
|
|
340
|
+
# Create temp directory
|
|
341
|
+
temp_dir = Path(tempfile.mkdtemp())
|
|
342
|
+
|
|
343
|
+
# Save screenshot and get filepath
|
|
344
|
+
filepath, _ = save_image_to_downloads(
|
|
345
|
+
screenshot, prefix="screenshot", directory=temp_dir
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
# Create Image object from filepath
|
|
349
|
+
image = Image(filepath)
|
|
350
|
+
|
|
351
|
+
if save_to_downloads:
|
|
352
|
+
log("Copying screenshot from temp to downloads")
|
|
353
|
+
shutil.copy(filepath, get_downloads_dir())
|
|
354
|
+
|
|
355
|
+
return image # MCP Image object
|
|
356
|
+
|
|
357
|
+
except Exception as e:
|
|
358
|
+
log(f"Error in screenshot or getting UI elements: {str(e)}")
|
|
359
|
+
import traceback
|
|
360
|
+
|
|
361
|
+
stack_trace = traceback.format_exc()
|
|
362
|
+
log(f"Stack trace:\n{stack_trace}")
|
|
363
|
+
return f"Error in screenshot or getting UI elements: {str(e)}\nStack trace:\n{stack_trace}"
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def is_low_spec_pc() -> bool:
|
|
367
|
+
try:
|
|
368
|
+
import psutil
|
|
369
|
+
|
|
370
|
+
cpu_low = psutil.cpu_count(logical=False) < 4
|
|
371
|
+
ram_low = psutil.virtual_memory().total < 8 * 1024**3
|
|
372
|
+
return cpu_low or ram_low
|
|
373
|
+
except Exception:
|
|
374
|
+
# Fallback if psutil not available or info unavailable
|
|
375
|
+
return False
|
|
376
|
+
|
|
215
377
|
|
|
378
|
+
@mcp.tool()
|
|
379
|
+
def take_screenshot_with_ocr(
|
|
380
|
+
title_pattern: str = None,
|
|
381
|
+
use_regex: bool = False,
|
|
382
|
+
threshold: int = 10,
|
|
383
|
+
scale_percent_for_ocr: int = None,
|
|
384
|
+
save_to_downloads: bool = False,
|
|
385
|
+
) -> str:
|
|
386
|
+
"""
|
|
387
|
+
Get OCR text from screenshot with absolute coordinates as JSON string of List[Tuple[List[List[int]], str, float]] (returned after adding the window offset from true (0, 0) of screen to the OCR coordinates, so clicking is on-point. Recommended to click in the middle of OCR Box) and using confidence from window with the specified title pattern.
|
|
388
|
+
If no title pattern is provided, get screenshot of entire screen and all text on the screen.
|
|
389
|
+
Know that OCR takes around 20 seconds on an mid spec pc at 1080p resolution.
|
|
390
|
+
|
|
391
|
+
Args:
|
|
392
|
+
title_pattern: Pattern to match window title, if None, take screenshot of entire screen
|
|
393
|
+
use_regex: If True, treat the pattern as a regex, otherwise best match with fuzzy matching
|
|
394
|
+
threshold: Minimum score (0-100) required for a fuzzy match
|
|
395
|
+
scale_percent_for_ocr: Percentage to scale the image down before processing, you wont need this most of the time unless your pc is extremely old or slow
|
|
396
|
+
save_to_downloads: If True, save the screenshot to the downloads directory and return the absolute path
|
|
397
|
+
|
|
398
|
+
Returns:
|
|
399
|
+
Returns a list of UI elements as List[Tuple[List[List[int]], str, float]] where each tuple is [[4 corners of box], text, confidence], "content type image not supported" means preview isnt supported but Image object is there.
|
|
400
|
+
"""
|
|
401
|
+
try:
|
|
216
402
|
all_windows = gw.getAllWindows()
|
|
217
403
|
|
|
218
404
|
# Convert to list of dictionaries for _find_matching_window
|
|
@@ -235,23 +421,24 @@ def take_screenshot(
|
|
|
235
421
|
# Take the screenshot
|
|
236
422
|
if not window:
|
|
237
423
|
log("No matching window found, taking screenshot of entire screen")
|
|
238
|
-
screenshot =
|
|
424
|
+
screenshot = _mss_screenshot()
|
|
239
425
|
else:
|
|
240
426
|
current_active_window = gw.getActiveWindow()
|
|
241
427
|
log(f"Taking screenshot of window: {window.title}")
|
|
242
428
|
# Activate the window and wait for it to be fully in focus
|
|
243
429
|
try:
|
|
244
|
-
|
|
245
430
|
window.activate()
|
|
246
431
|
pyautogui.sleep(0.5) # Wait for 0.5 seconds to ensure window is active
|
|
247
|
-
screenshot =
|
|
432
|
+
screenshot = _mss_screenshot(
|
|
248
433
|
region=(window.left, window.top, window.width, window.height)
|
|
249
434
|
)
|
|
250
435
|
# Restore the previously active window
|
|
251
436
|
if current_active_window:
|
|
252
437
|
try:
|
|
253
438
|
current_active_window.activate()
|
|
254
|
-
pyautogui.sleep(
|
|
439
|
+
pyautogui.sleep(
|
|
440
|
+
0.2
|
|
441
|
+
) # Wait a bit to ensure previous window is restored
|
|
255
442
|
except Exception as e:
|
|
256
443
|
log(f"Error restoring previous window: {str(e)}")
|
|
257
444
|
except Exception as e:
|
|
@@ -269,9 +456,6 @@ def take_screenshot(
|
|
|
269
456
|
# Create Image object from filepath
|
|
270
457
|
image = Image(filepath)
|
|
271
458
|
|
|
272
|
-
if not with_ocr_text_and_coords:
|
|
273
|
-
return image # MCP Image object
|
|
274
|
-
|
|
275
459
|
# Copy from temp to downloads
|
|
276
460
|
if save_to_downloads:
|
|
277
461
|
log("Copying screenshot from temp to downloads")
|
|
@@ -282,7 +466,7 @@ def take_screenshot(
|
|
|
282
466
|
|
|
283
467
|
if scale_percent_for_ocr is None:
|
|
284
468
|
# Calculate percent to scale height to 360 pixels
|
|
285
|
-
scale_percent_for_ocr = 100
|
|
469
|
+
scale_percent_for_ocr = 100 # 360 / img.shape[0] * 100
|
|
286
470
|
|
|
287
471
|
# Lower down resolution before processing
|
|
288
472
|
width = int(img.shape[1] * scale_percent_for_ocr / 100)
|
|
@@ -291,20 +475,28 @@ def take_screenshot(
|
|
|
291
475
|
resized_img = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
|
|
292
476
|
# save resized image to pwd
|
|
293
477
|
# cv2.imwrite("resized_img.png", resized_img)
|
|
294
|
-
|
|
478
|
+
|
|
295
479
|
output = engine(resized_img)
|
|
296
|
-
boxes
|
|
297
|
-
txts
|
|
480
|
+
boxes = output.boxes
|
|
481
|
+
txts = output.txts
|
|
298
482
|
scores = output.scores
|
|
299
483
|
zipped_results = list(zip(boxes, txts, scores))
|
|
300
484
|
zipped_results = [
|
|
301
|
-
(
|
|
485
|
+
(
|
|
486
|
+
box.tolist(),
|
|
487
|
+
text,
|
|
488
|
+
float(score),
|
|
489
|
+
) # convert np.array -> list, ensure score is float
|
|
302
490
|
for box, text, score in zipped_results
|
|
303
491
|
]
|
|
304
492
|
log(f"Found {len(zipped_results)} text items in OCR result.")
|
|
305
493
|
log(f"First 5 items: {zipped_results[:5]}")
|
|
306
|
-
return
|
|
307
|
-
|
|
494
|
+
return (
|
|
495
|
+
",\n".join([str(item) for item in zipped_results])
|
|
496
|
+
if zipped_results
|
|
497
|
+
else "No text found"
|
|
498
|
+
)
|
|
499
|
+
|
|
308
500
|
except Exception as e:
|
|
309
501
|
log(f"Error in screenshot or getting UI elements: {str(e)}")
|
|
310
502
|
import traceback
|
|
@@ -324,6 +516,26 @@ def move_mouse(x: int, y: int) -> str:
|
|
|
324
516
|
return f"Error moving mouse to coordinates ({x}, {y}): {str(e)}"
|
|
325
517
|
|
|
326
518
|
|
|
519
|
+
@mcp.tool()
|
|
520
|
+
def mouse_down(button: str = "left") -> str:
|
|
521
|
+
"""Hold down a mouse button ('left', 'right', 'middle')."""
|
|
522
|
+
try:
|
|
523
|
+
pyautogui.mouseDown(button=button)
|
|
524
|
+
return f"Held down {button} mouse button"
|
|
525
|
+
except Exception as e:
|
|
526
|
+
return f"Error holding {button} mouse button: {str(e)}"
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
@mcp.tool()
|
|
530
|
+
def mouse_up(button: str = "left") -> str:
|
|
531
|
+
"""Release a mouse button ('left', 'right', 'middle')."""
|
|
532
|
+
try:
|
|
533
|
+
pyautogui.mouseUp(button=button)
|
|
534
|
+
return f"Released {button} mouse button"
|
|
535
|
+
except Exception as e:
|
|
536
|
+
return f"Error releasing {button} mouse button: {str(e)}"
|
|
537
|
+
|
|
538
|
+
|
|
327
539
|
@mcp.tool()
|
|
328
540
|
async def drag_mouse(
|
|
329
541
|
from_x: int, from_y: int, to_x: int, to_y: int, duration: float = 0.5
|
|
@@ -353,14 +565,69 @@ async def drag_mouse(
|
|
|
353
565
|
return f"Error dragging from ({from_x}, {from_y}) to ({to_x}, {to_y}): {str(e)}"
|
|
354
566
|
|
|
355
567
|
|
|
568
|
+
import pyautogui
|
|
569
|
+
from typing import Union, List
|
|
570
|
+
|
|
571
|
+
|
|
356
572
|
@mcp.tool()
|
|
357
|
-
def
|
|
358
|
-
"""
|
|
573
|
+
def key_down(key: str) -> str:
|
|
574
|
+
"""Hold down a specific keyboard key until released."""
|
|
359
575
|
try:
|
|
360
|
-
pyautogui.
|
|
361
|
-
return f"
|
|
576
|
+
pyautogui.keyDown(key)
|
|
577
|
+
return f"Held down key: {key}"
|
|
578
|
+
except Exception as e:
|
|
579
|
+
return f"Error holding key {key}: {str(e)}"
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
@mcp.tool()
|
|
583
|
+
def key_up(key: str) -> str:
|
|
584
|
+
"""Release a specific keyboard key."""
|
|
585
|
+
try:
|
|
586
|
+
pyautogui.keyUp(key)
|
|
587
|
+
return f"Released key: {key}"
|
|
588
|
+
except Exception as e:
|
|
589
|
+
return f"Error releasing key {key}: {str(e)}"
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
@mcp.tool()
|
|
593
|
+
def press_keys(keys: Union[str, List[Union[str, List[str]]]]) -> str:
|
|
594
|
+
"""
|
|
595
|
+
Press keyboard keys.
|
|
596
|
+
|
|
597
|
+
Args:
|
|
598
|
+
keys:
|
|
599
|
+
- Single key as string (e.g., "enter")
|
|
600
|
+
- Sequence of keys as list (e.g., ["a", "b", "c"])
|
|
601
|
+
- Key combinations as nested list (e.g., [["ctrl", "c"], ["alt", "tab"]])
|
|
602
|
+
|
|
603
|
+
Examples:
|
|
604
|
+
press_keys("enter")
|
|
605
|
+
press_keys(["a", "b", "c"])
|
|
606
|
+
press_keys([["ctrl", "c"], ["alt", "tab"]])
|
|
607
|
+
"""
|
|
608
|
+
try:
|
|
609
|
+
if isinstance(keys, str):
|
|
610
|
+
# Single key
|
|
611
|
+
pyautogui.press(keys)
|
|
612
|
+
return f"Pressed single key: {keys}"
|
|
613
|
+
|
|
614
|
+
elif isinstance(keys, list):
|
|
615
|
+
for item in keys:
|
|
616
|
+
if isinstance(item, str):
|
|
617
|
+
# Sequential key press
|
|
618
|
+
pyautogui.press(item)
|
|
619
|
+
elif isinstance(item, list):
|
|
620
|
+
# Key combination (e.g., ctrl+c)
|
|
621
|
+
pyautogui.hotkey(*item)
|
|
622
|
+
else:
|
|
623
|
+
return f"Invalid key format: {item}"
|
|
624
|
+
return f"Successfully pressed keys sequence: {keys}"
|
|
625
|
+
|
|
626
|
+
else:
|
|
627
|
+
return "Invalid input: must be str or list"
|
|
628
|
+
|
|
362
629
|
except Exception as e:
|
|
363
|
-
return f"Error pressing
|
|
630
|
+
return f"Error pressing keys {keys}: {str(e)}"
|
|
364
631
|
|
|
365
632
|
|
|
366
633
|
@mcp.tool()
|
|
@@ -455,6 +722,7 @@ def main():
|
|
|
455
722
|
|
|
456
723
|
try:
|
|
457
724
|
# Run the server
|
|
725
|
+
log("Computer Control MCP Server Started...")
|
|
458
726
|
mcp.run()
|
|
459
727
|
|
|
460
728
|
except KeyboardInterrupt:
|
|
@@ -13,6 +13,8 @@ import tempfile
|
|
|
13
13
|
# --- Auto-install dependencies if needed ---
|
|
14
14
|
import pyautogui
|
|
15
15
|
from mcp.server.fastmcp import FastMCP, Image
|
|
16
|
+
import mss
|
|
17
|
+
from PIL import Image as PILImage
|
|
16
18
|
import pygetwindow as gw
|
|
17
19
|
from fuzzywuzzy import fuzz, process
|
|
18
20
|
|
|
@@ -39,6 +41,34 @@ def get_downloads_dir() -> Path:
|
|
|
39
41
|
return Path.home() / "Downloads"
|
|
40
42
|
|
|
41
43
|
|
|
44
|
+
def _mss_screenshot(region=None):
|
|
45
|
+
"""Take a screenshot using mss and return PIL Image.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
region: Optional tuple (left, top, width, height) for region capture
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
PIL Image object
|
|
52
|
+
"""
|
|
53
|
+
with mss.mss() as sct:
|
|
54
|
+
if region is None:
|
|
55
|
+
# Full screen screenshot
|
|
56
|
+
monitor = sct.monitors[0] # All monitors combined
|
|
57
|
+
else:
|
|
58
|
+
# Region screenshot
|
|
59
|
+
left, top, width, height = region
|
|
60
|
+
monitor = {
|
|
61
|
+
"left": left,
|
|
62
|
+
"top": top,
|
|
63
|
+
"width": width,
|
|
64
|
+
"height": height,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
screenshot = sct.grab(monitor)
|
|
68
|
+
# Convert to PIL Image
|
|
69
|
+
return PILImage.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
|
|
70
|
+
|
|
71
|
+
|
|
42
72
|
def save_image_to_downloads(
|
|
43
73
|
image, prefix: str = "screenshot", directory: Path = None
|
|
44
74
|
) -> Tuple[str, bytes]:
|
|
@@ -171,13 +201,13 @@ def take_screenshot(
|
|
|
171
201
|
# Take the screenshot
|
|
172
202
|
if not window:
|
|
173
203
|
print("No matching window found, taking screenshot of entire screen")
|
|
174
|
-
screenshot =
|
|
204
|
+
screenshot = _mss_screenshot()
|
|
175
205
|
else:
|
|
176
206
|
print(f"Taking screenshot of window: {window.title}")
|
|
177
207
|
# Activate the window and wait for it to be fully in focus
|
|
178
208
|
window.activate()
|
|
179
209
|
pyautogui.sleep(0.5) # Wait for 0.5 seconds to ensure window is active
|
|
180
|
-
screenshot =
|
|
210
|
+
screenshot = _mss_screenshot(
|
|
181
211
|
region=(window.left, window.top, window.width, window.height)
|
|
182
212
|
)
|
|
183
213
|
# Restore the previously active window
|
|
@@ -252,13 +282,13 @@ def get_ocr_from_screenshot(
|
|
|
252
282
|
# Take the screenshot
|
|
253
283
|
if not window:
|
|
254
284
|
log("No matching window found, taking screenshot of entire screen")
|
|
255
|
-
screenshot =
|
|
285
|
+
screenshot = _mss_screenshot()
|
|
256
286
|
else:
|
|
257
287
|
log(f"Taking screenshot of window: {window.title}")
|
|
258
288
|
# Activate the window and wait for it to be fully in focus
|
|
259
289
|
window.activate()
|
|
260
290
|
pyautogui.sleep(0.5) # Wait for 0.5 seconds to ensure window is active
|
|
261
|
-
screenshot =
|
|
291
|
+
screenshot = _mss_screenshot(
|
|
262
292
|
region=(window.left, window.top, window.width, window.height)
|
|
263
293
|
)
|
|
264
294
|
# Restore the previously active window
|
|
@@ -1,48 +1,50 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["hatchling"]
|
|
3
|
-
build-backend = "hatchling.build"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "computer-control-mcp"
|
|
7
|
-
version = "0.3.
|
|
8
|
-
description = "MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.10"
|
|
11
|
-
license = {text = "MIT"}
|
|
12
|
-
authors = [{name = "AB498", email = "abcd49800@gmail.com"}]
|
|
13
|
-
classifiers = [
|
|
14
|
-
"Development Status :: 4 - Beta",
|
|
15
|
-
"Intended Audience :: Developers",
|
|
16
|
-
"License :: OSI Approved :: MIT License",
|
|
17
|
-
"Programming Language :: Python :: 3",
|
|
18
|
-
"Programming Language :: Python :: 3.12",
|
|
19
|
-
"Topic :: Software Development :: Libraries",
|
|
20
|
-
"Topic :: Utilities"
|
|
21
|
-
]
|
|
22
|
-
dependencies = [
|
|
23
|
-
"pyautogui==0.9.54",
|
|
24
|
-
"mcp[cli]==1.13.0",
|
|
25
|
-
"pillow==11.3.0",
|
|
26
|
-
"pygetwindow==0.0.9",
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
[
|
|
48
|
-
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "computer-control-mcp"
|
|
7
|
+
version = "0.3.3"
|
|
8
|
+
description = "MCP server that provides computer control capabilities, like mouse, keyboard, OCR, etc. using PyAutoGUI, RapidOCR, ONNXRuntime. Similar to 'computer-use' by Anthropic. With Zero External Dependencies."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "AB498", email = "abcd49800@gmail.com"}]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 4 - Beta",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Topic :: Software Development :: Libraries",
|
|
20
|
+
"Topic :: Utilities"
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"pyautogui==0.9.54",
|
|
24
|
+
"mcp[cli]==1.13.0",
|
|
25
|
+
"pillow==11.3.0",
|
|
26
|
+
"pygetwindow==0.0.9",
|
|
27
|
+
"pywinctl==0.4.1",
|
|
28
|
+
"fuzzywuzzy==0.18.0",
|
|
29
|
+
"rapidocr==3.3.1",
|
|
30
|
+
"onnxruntime==1.22.0",
|
|
31
|
+
"rapidocr_onnxruntime==1.2.3",
|
|
32
|
+
"opencv-python==4.12.0.88",
|
|
33
|
+
"mss>=7.0.0"
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/AB498/computer-control-mcp"
|
|
38
|
+
Issues = "https://github.com/AB498/computer-control-mcp/issues"
|
|
39
|
+
Documentation = "https://github.com/AB498/computer-control-mcp#readme"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
computer-control-mcp = "computer_control_mcp.cli:main"
|
|
43
|
+
computer-control-mcp-server = "computer_control_mcp.server:main"
|
|
44
|
+
|
|
45
|
+
[tool.hatch.build]
|
|
46
|
+
sources = ["src"]
|
|
47
|
+
packages = ["src/computer_control_mcp"]
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.wheel]
|
|
50
|
+
packages = ["src/computer_control_mcp"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{computer_control_mcp-0.3.0 → computer_control_mcp-0.3.3}/computer_control_mcp/test_image.png
RENAMED
|
File without changes
|