AutoMonkey 0.4.1__py3-none-any.whl

automonkey/__init__.py

@@ -0,0 +1,37 @@
+"""Top-level package for AutoMonkey."""
+
+__author__ = """Mihail-Cosmin Munteanu"""
+__email__ = 'munteanumihailcosmin@gmail.com'
+# Read back from the installed distribution, so [project].version in pyproject.toml stays
+# the only place the version is written (see bump_version.py).
+try:
+    from importlib.metadata import version as _installed_version
+
+    __version__ = _installed_version("AutoMonkey")
+except Exception:  # plain source checkout, no installed distribution metadata to read
+    __version__ = "0.0.0+unknown"
+
+
+def _default_display_env():
+    """Fill in display/session variables missing when started by a service or an MCP
+    client: pyautogui fails to import without DISPLAY, and the AT-SPI backend
+    (atspi_funcs.py) can't reach the accessibility bus without DBUS_SESSION_BUS_ADDRESS -
+    both are normally set by a login session, which a service/MCP client may not inherit."""
+    import os
+    import sys
+    if not sys.platform.startswith("linux"):
+        return
+    runtime = os.environ.setdefault("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")
+    if "WAYLAND_DISPLAY" not in os.environ and os.path.exists(os.path.join(runtime, "wayland-0")):
+        os.environ["WAYLAND_DISPLAY"] = "wayland-0"
+    if "DISPLAY" not in os.environ and os.path.exists("/tmp/.X11-unix/X0"):
+        os.environ["DISPLAY"] = ":0"
+    if "DBUS_SESSION_BUS_ADDRESS" not in os.environ:
+        bus_path = os.path.join(runtime, "bus")
+        if os.path.exists(bus_path):
+            os.environ["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={bus_path}"
+
+
+_default_display_env()
+
+from .automonkey import *

automonkey/ai/__init__.py

@@ -0,0 +1,54 @@
+"""AI-friendly layer over AutoMonkey: act + observe in a single call.
+
+    from automonkey.ai import act, observe
+    text, img = act("launch firefox; maximize; url google.ro; type betano.ro")
+
+All coordinates are pixels of the screenshot the AI was shown.
+"""
+from __future__ import annotations
+
+from PIL import Image
+
+from . import ocr, screen, windows
+from .script import run, split_script
+
+
+def _context() -> str:
+    v = screen.view()
+    try:
+        w = windows.active_window()
+        active = f"active window: {w['title']!r}"
+    except Exception:
+        active = "active window: unknown"
+    return f"{v.describe()}; {active}"
+
+
+def observe(grid: int = 0, max_side: int = screen.DEFAULT_MAX_SIDE) -> tuple[str, Image.Image]:
+    img = screen.shot(grid=grid, max_side=max_side)
+    return _context(), img
+
+
+def act(script: str, screenshot: bool = True, grid: int = 0, settle: bool = True) -> tuple[str, Image.Image | None]:
+    """Run an action script, then report what happened (and show the screen)."""
+    before = screen.grab()
+    log, ok = run(script, settle=settle)
+    after = screen.grab()
+    img = screen.shot(grid=grid, full=after) if screenshot else None
+    lines = log + [screen.diff_summary(before, after), _context()]
+    if not ok:
+        lines.insert(0, "STOPPED AT A FAILED STEP")
+    return "\n".join(lines), img
+
+
+def window_list() -> str:
+    v = screen.view()
+    out = []
+    for w in windows.list_windows():
+        x, y = v.to_image(w["x"], w["y"])
+        r, b = v.to_image(w["x"] + w["w"], w["y"] + w["h"])
+        out.append(f"{'*' if w['active'] else ' '} {w['id']:>10}  {w['title']!r}  [{w['wm_class']}]  "
+                   f"x={x}..{r} y={y}..{b}")
+    return "\n".join(out) or "no windows"
+
+
+__all__ = ["act", "observe", "window_list", "ocr", "screen", "windows", "run", "split_script"]

automonkey/ai/__main__.py

@@ -0,0 +1,76 @@
+"""Command line for agents that only have a shell (one command = act + screenshot).
+
+    python -m automonkey.ai shot [--grid 100]
+    python -m automonkey.ai do "launch firefox; maximize; url google.ro"
+    python -m automonkey.ai zoom X Y W H
+    python -m automonkey.ai windows
+    python -m automonkey.ai find "Accept all"
+    python -m automonkey.ai read [X Y W H]
+
+Screenshots are written to --out (default: $AUTOMONKEY_AI_OUT or
+/tmp/automonkey_ai/screen.png) and the path is printed; open it to look.
+Coordinates are pixels of that screenshot.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import tempfile
+
+from . import act, observe, ocr, screen, window_list
+
+DEFAULT_OUT = os.environ.get("AUTOMONKEY_AI_OUT",
+                             os.path.join(tempfile.gettempdir(), "automonkey_ai", "screen.png"))
+
+
+def _save(img, out: str) -> str:
+    os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
+    img.save(out, compress_level=1)
+    return out
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="python -m automonkey.ai", description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--out", default=DEFAULT_OUT, help="where to save the screenshot")
+    p.add_argument("--grid", type=int, default=0, help="overlay a labelled grid every N pixels")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("shot", help="take a screenshot")
+    d = sub.add_parser("do", help="run an action script, then take a screenshot")
+    d.add_argument("script", help="steps separated by ';' or newlines ('-' reads stdin)")
+    d.add_argument("--no-shot", action="store_true", help="don't take a screenshot")
+    z = sub.add_parser("zoom", help="enlarged view of a region")
+    z.add_argument("box", type=int, nargs=4, metavar=("X", "Y", "W", "H"))
+    sub.add_parser("windows", help="list windows")
+    f = sub.add_parser("find", help="find text on screen (OCR)")
+    f.add_argument("text")
+    r = sub.add_parser("read", help="read text on screen (OCR)")
+    r.add_argument("box", type=int, nargs="*", metavar="X Y W H")
+    a = p.parse_args(argv)
+
+    if a.cmd == "shot":
+        text, img = observe(grid=a.grid)
+        print(text)
+        print("screenshot:", _save(img, a.out))
+    elif a.cmd == "do":
+        script = sys.stdin.read() if a.script == "-" else a.script
+        text, img = act(script, screenshot=not a.no_shot, grid=a.grid)
+        print(text)
+        if img is not None:
+            print("screenshot:", _save(img, a.out))
+        return 0 if not text.startswith("STOPPED") else 1
+    elif a.cmd == "zoom":
+        print("zoom:", _save(screen.zoom(*a.box, grid=a.grid), a.out.replace(".png", "_zoom.png")))
+    elif a.cmd == "windows":
+        print(window_list())
+    elif a.cmd == "find":
+        hits = ocr.find_text(a.text)
+        print("\n".join(f"{h['text']!r} click at {h['cx']},{h['cy']}" for h in hits) or "not found")
+    elif a.cmd == "read":
+        print(ocr.read_text(*(a.box or [0, 0, 0, 0])))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

automonkey/ai/mcp_launch.py

@@ -0,0 +1,62 @@
+"""Start the MCP server with the project's virtualenv, on any OS.
+
+A venv keeps its interpreter in `.venv/bin/python` on Linux/macOS but in
+`.venv\\Scripts\\python.exe` on Windows, and `.mcp.json` can't branch on the OS. So
+`.mcp.json` runs this file with whatever Python is at hand (stdlib only, any 3.x);
+it finds the venv's interpreter and hands over to the real server.
+
+    python3 automonkey/ai/mcp_launch.py           # start the server
+    python3 automonkey/ai/mcp_launch.py --which   # print the interpreter it would use
+
+Looks in $AUTOMONKEY_VENV, then <repo>/.venv, <repo>/venv, then $VIRTUAL_ENV; falls
+back to the running interpreter if it can import the server.
+"""
+import importlib.util
+import os
+import subprocess
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def venv_python():
+    for venv in (os.environ.get("AUTOMONKEY_VENV"), os.path.join(ROOT, ".venv"), os.path.join(ROOT, "venv"),
+                 os.environ.get("VIRTUAL_ENV")):
+        if not venv:
+            continue
+        for exe in (os.path.join(venv, "Scripts", "python.exe"), os.path.join(venv, "bin", "python")):
+            if os.path.isfile(exe):
+                return exe
+    # The server runs with cwd=ROOT, so the package comes from there; only its dependencies
+    # have to be installed in this interpreter.
+    server = os.path.join(ROOT, "automonkey", "ai", "mcp_server.py")
+    if os.path.isfile(server) and all(importlib.util.find_spec(m) for m in ("mcp", "pyautogui")):
+        return sys.executable
+    return None
+
+
+def main():
+    python = venv_python()
+    if not python:
+        pip = r".venv\Scripts\pip" if sys.platform == "win32" else ".venv/bin/pip"
+        sys.stderr.write("automonkey MCP: no virtualenv found in {}. Create it:\n"
+                         "  python -m venv .venv && {} install -e .[ai]\n"
+                         "or point AUTOMONKEY_VENV at an existing one.\n".format(ROOT, pip))
+        return 1
+    if "--which" in sys.argv:
+        print(python)
+        return 0
+    cmd = [python, "-m", "automonkey.ai.mcp_server"]
+    if sys.platform != "win32":
+        os.chdir(ROOT)
+        os.execv(python, cmd)  # replace this process: signals and stdio go straight to the server
+    # Windows has no real exec (os.execv spawns and exits, detaching stdio), so run it as a
+    # child that inherits our stdin/stdout pipes, and pass its exit code on.
+    try:
+        return subprocess.call(cmd, cwd=ROOT)
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

automonkey/ai/mcp_server.py

@@ -0,0 +1,114 @@
+"""MCP server exposing AutoMonkey to Claude (or any MCP client).
+
+The repo's .mcp.json starts it through mcp_launch.py, which finds the venv on any OS.
+Register it by hand with Claude Code (from the repo root):
+    claude mcp add automonkey -- python3 "$PWD/automonkey/ai/mcp_launch.py"
+"""
+from __future__ import annotations
+
+from mcp.server.mcpserver import Image, MCPServer
+
+from . import act as _act
+from . import observe, ocr, screen, window_list
+from .script import __doc__ as SCRIPT_DOC
+from .. import list_controls as _list_controls
+
+INSTRUCTIONS = f"""Control this computer's screen, mouse and keyboard.
+
+Workflow: call `act` with a whole script of steps; it returns a log, a summary of what
+changed on screen, and a fresh screenshot -- you rarely need a separate `screenshot`
+call. Batch everything you are confident about into one `act` call; only stop to look
+when the next step depends on what appears.
+
+Coordinates are ALWAYS pixels of the latest screenshot image you were shown (the
+server converts to real screen pixels). The red ring marks the mouse pointer.
+
+Tips: use `url` instead of clicking the address bar; `launch` an app rather than
+clicking desktop icons; `zoom` to read small text precisely; `grid=100` overlays
+labelled lines when you need exact coordinates; dismiss unexpected popups with
+`key escape`. OCR tools (find_text/read_text/click_text) are
+{'available' if ocr.available() else 'NOT available yet (tesseract not found); they start working without a restart once it is installed'}.
+
+Action script language:
+{SCRIPT_DOC}"""
+
+mcp = MCPServer("automonkey", instructions=INSTRUCTIONS)
+
+
+def _img(pil) -> Image:
+    return Image(data=screen.to_png(pil), format="png")
+
+
+@mcp.tool()
+def act(script: str, screenshot: bool = True, grid: int = 0) -> list:
+    """Run an action script (one command per line or ';'-separated), wait for the screen to
+    settle, then return the step log, what changed, and a screenshot.
+
+    Commands: click X Y [button] [N] | dclick X Y | rclick X Y | move X Y | drag X1 Y1 X2 Y2 |
+    scroll N [X Y] | type TEXT | key K [K..] | hotkey ctrl+l | wait S | stable [T] | change [T] |
+    url ADDRESS | launch CMD | focus WINDOW | maximize [WINDOW] | close WINDOW |
+    click_text TEXT | wait_text TEXT | click_image PATH | press [BUTTON] | release [BUTTON] |
+    path X1 Y1 X2 Y2 .. (freehand: press, glide through the points, release)
+
+    Example: "launch firefox; maximize; url google.ro; click 640 290; type betano.ro"
+    Set screenshot=false when you don't need to look (saves tokens); grid=100 adds a grid."""
+    text, img = _act(script, screenshot=screenshot, grid=grid)
+    return [text, _img(img)] if img is not None else [text]
+
+
+@mcp.tool()
+def screenshot(grid: int = 0) -> list:
+    """Take a screenshot. grid=N overlays labelled lines every N pixels for precise coordinates."""
+    text, img = observe(grid=grid)
+    return [text, _img(img)]
+
+
+@mcp.tool()
+def zoom(x: int, y: int, width: int, height: int, grid: int = 0) -> list:
+    """Enlarged full-resolution view of a screenshot region (screenshot coordinates), for small
+    text or precise targeting. Grid labels, if requested, use the same screenshot coordinates."""
+    return [f"zoom of x={x}..{x + width} y={y}..{y + height}", _img(screen.zoom(x, y, width, height, grid))]
+
+
+@mcp.tool()
+def windows() -> str:
+    """List open windows ('*' = active) with ids, titles and their screenshot-coordinate bounds."""
+    return window_list()
+
+
+@mcp.tool()
+def list_controls(window: str, depth: int = None) -> str:
+    """List a window's controls via the OS accessibility tree (pywinauto on Windows,
+    AT-SPI on Linux; not implemented on macOS yet) - role, name and the dict to pass as
+    `control` to click/type/etc. Use this instead of screenshots/OCR when the app exposes
+    an accessibility tree (most GTK/Qt/native apps do; Electron needs
+    --force-renderer-accessibility, canvas-drawn UIs don't expose one at all).
+
+    window: title of the top-level window, regex matched anywhere in the title.
+    depth: how many levels deep to go (default: all)."""
+    return _list_controls(window, depth)
+
+
+@mcp.tool()
+def find_text(text: str) -> str:
+    """Locate text on screen via OCR. Returns boxes and click points in screenshot coordinates."""
+    hits = ocr.find_text(text)
+    if not hits:
+        return f"{text!r} not found"
+    return "\n".join(f"{h['text']!r} click at {h['cx']},{h['cy']} (box x={h['x']} y={h['y']} w={h['w']} h={h['h']})"
+                     for h in hits[:10])
+
+
+@mcp.tool()
+def read_text(x: int = 0, y: int = 0, width: int = 0, height: int = 0) -> str:
+    """OCR the screen (or a region in screenshot coordinates) and return the text. Cheaper than
+    a screenshot when you only need to read."""
+    return ocr.read_text(x, y, width, height) or "(no text found)"
+
+
+def main():
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

automonkey/ai/ocr.py

@@ -0,0 +1,149 @@
+"""Optional OCR: find text on screen so the AI can click by label instead of coordinates.
+
+Needs the tesseract binary. Installation differs per OS:
+  Linux   `sudo apt install tesseract-ocr` (add tesseract-ocr-ron for Romanian diacritics)
+  macOS   `brew install tesseract`
+  Windows `winget install UB-Mannheim.TesseractOCR`
+
+The binary is located via $TESSERACT_CMD, then PATH, then (Windows) the PATH saved
+in the registry, then the usual install directories -- so OCR works even when the
+PATH of the process that launched us predates the tesseract install (e.g. an
+already-running MCP server, or a macOS GUI app that never sees Homebrew's PATH).
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+
+from PIL import Image, ImageOps
+
+from .screen import grab, view
+
+def _candidates() -> list[str]:
+    if sys.platform == "win32":
+        env = os.environ.get
+        dirs = [env("ProgramFiles", r"C:\Program Files"), env("ProgramFiles(x86)", r"C:\Program Files (x86)"),
+                os.path.join(env("LOCALAPPDATA", ""), "Programs")]  # per-user installs
+        return ([os.path.join(d, "Tesseract-OCR", "tesseract.exe") for d in dirs]
+                + [os.path.join(env("USERPROFILE", ""), "scoop", "shims", "tesseract.exe"),
+                   os.path.join(env("ChocolateyInstall", r"C:\ProgramData\chocolatey"), "bin", "tesseract.exe")])
+    return ["/opt/homebrew/bin/tesseract", "/usr/local/bin/tesseract", "/opt/local/bin/tesseract",
+            "/usr/bin/tesseract", "/snap/bin/tesseract"]
+
+
+def _registry_path() -> str:
+    """PATH as currently saved in the registry: installers update it there, but a running
+    process keeps the PATH it started with."""
+    import winreg
+    parts = []
+    for hive, key in ((winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment"),
+                      (winreg.HKEY_CURRENT_USER, "Environment")):
+        try:
+            with winreg.OpenKey(hive, key) as k:
+                parts.append(os.path.expandvars(winreg.QueryValueEx(k, "Path")[0]))
+        except OSError:
+            pass
+    return os.pathsep.join(parts)
+
+
+_found: str | None = None
+
+
+def tesseract_cmd() -> str | None:
+    """Locate the tesseract binary, or None if it isn't installed. $TESSERACT_CMD, when
+    set, is used exactly (a path or a command name); a wrong value is not silently
+    replaced by another tesseract."""
+    global _found
+    env = os.environ.get("TESSERACT_CMD")
+    if env:
+        return shutil.which(env)  # also accepts a full path, and adds .exe on Windows
+    if _found and os.path.isfile(_found):
+        return _found
+    _found = shutil.which("tesseract")
+    if not _found and sys.platform == "win32":
+        _found = shutil.which("tesseract", path=_registry_path())
+    if not _found:
+        _found = next((c for c in _candidates() if os.path.isfile(c)), None)
+    return _found
+
+
+def available() -> bool:
+    return tesseract_cmd() is not None
+
+
+def _require():
+    if available():
+        return
+    env = os.environ.get("TESSERACT_CMD")
+    if env:
+        raise RuntimeError(f"OCR: TESSERACT_CMD={env!r} is not an executable tesseract")
+    if sys.platform == "win32":
+        hint = "winget install UB-Mannheim.TesseractOCR (or set TESSERACT_CMD)"
+    elif sys.platform == "darwin":
+        hint = "brew install tesseract"
+    else:
+        hint = "sudo apt install tesseract-ocr (add tesseract-ocr-ron for Romanian diacritics)"
+    raise RuntimeError(f"OCR needs tesseract: {hint}")
+
+
+def _pytesseract():
+    import pytesseract
+    pytesseract.pytesseract.tesseract_cmd = tesseract_cmd()
+    return pytesseract
+
+
+def _words(img: Image.Image, lang: str):
+    pytesseract = _pytesseract()
+    gray = ImageOps.grayscale(img)
+    # Light-on-dark UIs (dark themes) OCR much better when inverted.
+    if sum(gray.resize((64, 64)).getdata()) / 4096 < 110:
+        gray = ImageOps.invert(gray)
+    data = pytesseract.image_to_data(gray, lang=lang, output_type=pytesseract.Output.DICT)
+    for i, text in enumerate(data["text"]):
+        if text.strip() and float(data["conf"][i]) > 30:
+            yield dict(text=text, x=data["left"][i], y=data["top"][i], w=data["width"][i], h=data["height"][i],
+                       line=(data["block_num"][i], data["par_num"][i], data["line_num"][i]))
+
+
+def find_text(query: str, lang: str = "eng") -> list[dict]:
+    """Every on-screen occurrence of `query` (case-insensitive, may span several words).
+    Returns boxes in image coordinates with a click point (cx, cy), best match first."""
+    _require()
+    v = view()
+    full = grab()
+    words = list(_words(full, lang))
+    q = query.lower().split()
+    hits = []
+    for i in range(len(words)):
+        group = words[i:i + len(q)]
+        if len(group) < len(q) or len({w["line"] for w in group}) > 1:
+            continue
+        joined = " ".join(w["text"].lower() for w in group)
+        if " ".join(q) in joined:
+            x0 = min(w["x"] for w in group)
+            y0 = min(w["y"] for w in group)
+            x1 = max(w["x"] + w["w"] for w in group)
+            y1 = max(w["y"] + w["h"] for w in group)
+            (ix0, iy0), (ix1, iy1) = v.capture_to_image(x0, y0), v.capture_to_image(x1, y1)
+            exact = joined == " ".join(q)
+            hits.append(dict(text=" ".join(w["text"] for w in group), x=ix0, y=iy0, w=ix1 - ix0, h=iy1 - iy0,
+                             cx=(ix0 + ix1) // 2, cy=(iy0 + iy1) // 2, exact=exact))
+    hits.sort(key=lambda h: (not h["exact"], h["y"], h["x"]))
+    return hits
+
+
+def read_text(x: int = 0, y: int = 0, w: int = 0, h: int = 0, lang: str = "eng") -> str:
+    """OCR the whole screen, or an image-coordinate region. Much cheaper for the AI
+    than reading text off a screenshot."""
+    _require()
+    pytesseract = _pytesseract()
+    v = view()
+    img = grab()
+    if w and h:
+        s = 1 / v.scale
+        img = img.crop((round(x * s), round(y * s), round((x + w) * s), round((y + h) * s)))
+    gray = ImageOps.grayscale(img)
+    if sum(gray.resize((64, 64)).getdata()) / 4096 < 110:
+        gray = ImageOps.invert(gray)
+    return pytesseract.image_to_string(gray, lang=lang).strip()