seedcode-cli 6.1.5__py3-none-any.whl

seedcode/computer/dpi.py

@@ -0,0 +1,185 @@
+"""DPI / display-scaling resilience for the Computer Engine.
+
+The engine drives three subsystems that each report coordinates in their own
+space unless told otherwise:
+
+* **UI Automation** (``uiautomation``) — ``BoundingRectangle`` is in *physical*
+  pixels once the process is per-monitor DPI aware, otherwise virtualized.
+* **mss** (screenshots) — always captures *physical* pixels.
+* **pyautogui** (mouse/keyboard) — operates in *physical* pixels only when the
+  process is DPI aware; otherwise Windows silently rescales its clicks, so a
+  point read from UIA/mss lands in the wrong place on any display scaled above
+  100 %.
+
+The fix is to declare the process **Per-Monitor-DPI-Aware v2** exactly once, as
+early as possible. Then all three agree on one physical-pixel space and a
+coordinate resolved from the accessibility tree or a screenshot can be clicked
+verbatim — on 4K laptops at 150 %, mixed-DPI multi-monitor rigs, and 100 %
+desktops alike.
+
+Everything here is Windows-only and defensive: on any other platform, or if the
+Win32 calls are unavailable, the functions degrade to safe no-ops so the rest of
+Seed Code keeps working. Nothing in this module touches an AI provider.
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from functools import lru_cache
+
+# Windows DPI-awareness context handles (see SetProcessDpiAwarenessContext).
+_DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = -4
+_DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE = -3
+# PROCESS_DPI_AWARENESS values (SetProcessDpiAwareness, Win 8.1+).
+_PROCESS_PER_MONITOR_DPI_AWARE = 2
+# The standard baseline DPI Windows scales everything relative to.
+BASELINE_DPI = 96
+
+_applied: str | None = None
+
+
+@dataclass(slots=True)
+class DpiInfo:
+    """The DPI-awareness state the engine achieved, for diagnostics."""
+
+    mode: str          # e.g. "per_monitor_v2", "per_monitor", "system", "none"
+    aware: bool        # whether physical-pixel alignment can be trusted
+    platform: str
+
+    def describe(self) -> str:
+        if not self.aware:
+            return f"DPI awareness: {self.mode} (coordinates may be virtualized)"
+        return f"DPI awareness: {self.mode} (physical-pixel aligned)"
+
+
+def ensure_dpi_awareness() -> DpiInfo:
+    """Declare this process per-monitor-DPI-aware v2 — idempotent, best-effort.
+
+    Called once at engine start (and safe to call again). Tries the strongest
+    awareness Windows offers, falling back through older APIs on downlevel
+    systems. If awareness was already set by the host process (an embedding
+    app, a manifest), Windows refuses the change and we simply report the
+    current state rather than fighting it.
+    """
+    global _applied
+    if _applied is not None:
+        return _current_info(_applied)
+
+    if sys.platform != "win32":
+        _applied = "none"
+        return _current_info("none")
+
+    mode = _apply_windows_awareness()
+    _applied = mode
+    return _current_info(mode)
+
+
+def _apply_windows_awareness() -> str:
+    """Walk the DPI-awareness APIs newest-first; return the mode that stuck."""
+    import ctypes
+
+    # 1) Windows 10 1703+ : richest, allows Per-Monitor v2 (dialogs, non-client
+    #    areas, and child windows all scale correctly).
+    try:
+        user32 = ctypes.windll.user32
+        if user32.SetProcessDpiAwarenessContext(
+            ctypes.c_void_p(_DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)
+        ):
+            return "per_monitor_v2"
+        if user32.SetProcessDpiAwarenessContext(
+            ctypes.c_void_p(_DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE)
+        ):
+            return "per_monitor"
+    except (AttributeError, OSError):
+        pass
+
+    # 2) Windows 8.1+ : per-monitor awareness via shcore.
+    try:
+        ctypes.windll.shcore.SetProcessDpiAwareness(_PROCESS_PER_MONITOR_DPI_AWARE)
+        return "per_monitor"
+    except (AttributeError, OSError):
+        # E_ACCESSDENIED here means awareness was already set for the process;
+        # treat that as success and read the real state below.
+        pass
+
+    # 3) Vista+ : system-DPI aware (single global scale factor).
+    try:
+        if ctypes.windll.user32.SetProcessDPIAware():
+            return "system"
+    except (AttributeError, OSError):
+        pass
+
+    return _detect_existing_mode()
+
+
+def _detect_existing_mode() -> str:
+    """Read the process's current awareness when we could not set it."""
+    if sys.platform != "win32":
+        return "none"
+    import ctypes
+
+    try:
+        awareness = ctypes.c_int()
+        ctypes.windll.shcore.GetProcessDpiAwareness(0, ctypes.byref(awareness))
+        return {0: "none", 1: "system", 2: "per_monitor"}.get(
+            awareness.value, "unknown"
+        )
+    except (AttributeError, OSError):
+        return "unknown"
+
+
+def _current_info(mode: str) -> DpiInfo:
+    aware = mode in ("per_monitor_v2", "per_monitor", "system")
+    return DpiInfo(mode=mode, aware=aware, platform=sys.platform)
+
+
+@lru_cache(maxsize=16)
+def scale_for_point(x: int, y: int) -> float:
+    """The DPI scale factor (1.0 == 100 %) of the monitor containing a point.
+
+    Used to translate physical-pixel geometry to/from a display's logical
+    layout when a driver reports logical coordinates. Returns 1.0 whenever the
+    real factor cannot be determined, so callers never divide by an unknown.
+    """
+    dpi = _dpi_for_point(x, y)
+    return (dpi / BASELINE_DPI) if dpi else 1.0
+
+
+def _dpi_for_point(x: int, y: int) -> int | None:
+    if sys.platform != "win32":
+        return None
+    import ctypes
+
+    try:
+        # MonitorFromPoint(pt, MONITOR_DEFAULTTONEAREST=2)
+        pt = _Point(int(x), int(y))
+        monitor = ctypes.windll.user32.MonitorFromPoint(pt, 2)
+        dpi_x = ctypes.c_uint()
+        dpi_y = ctypes.c_uint()
+        # GetDpiForMonitor(hmon, MDT_EFFECTIVE_DPI=0, &x, &y)
+        if ctypes.windll.shcore.GetDpiForMonitor(
+            monitor, 0, ctypes.byref(dpi_x), ctypes.byref(dpi_y)
+        ) == 0:
+            return int(dpi_x.value)
+    except (AttributeError, OSError):
+        return None
+    return None
+
+
+if sys.platform == "win32":
+    import ctypes
+
+    class _Point(ctypes.Structure):
+        _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)]
+else:  # pragma: no cover - non-Windows stub so imports never fail
+    class _Point:  # type: ignore[no-redef]
+        def __init__(self, x: int, y: int) -> None:
+            self.x, self.y = x, y
+
+
+def reset_for_tests() -> None:
+    """Clear the cached applied-state (test hook only)."""
+    global _applied
+    _applied = None
+    scale_for_point.cache_clear()

seedcode/computer/engine.py

@@ -0,0 +1,105 @@
+"""Computer Engine facade: the stable, offline, AI-independent entry point.
+
+This is the single object the rest of Seed Code talks to. It owns and wires the
+deterministic subsystems — controller (drivers), element resolver, state
+manager, verification engine, recovery engine and the skill dispatcher — and
+exposes a small, stable API:
+
+* :meth:`run_skill`  — run a named skill (or ``ui_*`` semantic verb) end-to-end
+  with verification and recovery; returns a :class:`DispatchResult`.
+* :meth:`state`      — the current :class:`ComputerState` (engine memory).
+* :meth:`see`        — a semantic snapshot for the AI to replan against.
+* :meth:`catalog`    — the skill manifest text for the AI, filtered by level.
+
+The engine contains no provider/AI code and works offline. It is created once
+per session (lazily) and holds no per-request state beyond the shared
+StateManager.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..tools.permissions import PermissionLevel
+from . import catalog as _catalog  # noqa: F401 — import populates the skill registry
+from .dispatcher import DispatchResult, SkillDispatcher
+from .recovery import RecoveryEngine
+from .resolver import ElementResolver
+from .skills import REGISTRY
+from .state import ComputerState, StateManager
+from .verifier import VerificationEngine
+
+
+class ComputerEngine:
+    """The deterministic hands, eyes, memory and reflexes of Seed Code."""
+
+    def __init__(self, permissions: Any, controller: Any = None) -> None:
+        self._permissions = permissions
+        if controller is None:
+            from .controller import ComputerController
+
+            controller = ComputerController()
+        self._controller = controller
+
+        # Subsystems, all deterministic and offline.
+        self._resolver = ElementResolver(
+            vision=getattr(controller, "vision", None),
+            screen=getattr(controller, "screen", None),
+        )
+        self._state = StateManager(
+            windows=getattr(controller, "windows", None),
+            mouse=getattr(controller, "mouse", None),
+        )
+        self._verifier = VerificationEngine(
+            vision=getattr(controller, "vision", None),
+            windows=getattr(controller, "windows", None),
+        )
+        self._recovery = RecoveryEngine(controller=controller, state=self._state)
+        self._dispatcher = SkillDispatcher(
+            controller=controller,
+            resolver=self._resolver,
+            state=self._state,
+            permissions=permissions,
+            registry=REGISTRY,
+            verifier=self._verifier,
+            recovery=self._recovery,
+        )
+
+    # --- public API ----------------------------------------------------------
+    def run_skill(
+        self,
+        name: str,
+        params: dict[str, Any] | None = None,
+        expected: dict[str, Any] | None = None,
+    ) -> DispatchResult:
+        """Execute a skill / semantic UI verb with verification and recovery."""
+        # Keep engine memory fresh before acting so state-aware skills are right.
+        self._state.refresh()
+        # Record what we're doing so the AI has continuity without re-asking.
+        self._state.set_task(f"{name} {params or ''}".strip())
+        result = self._dispatcher.dispatch(name, params, expected)
+        # The task is finished (successfully or not); the trail keeps the detail.
+        self._state.set_task(None)
+        return result
+
+    def state(self) -> ComputerState:
+        """Current engine memory (refreshed from the live desktop)."""
+        return self._state.refresh()
+
+    def see(self, window_title: str | None = None) -> str:
+        """A semantic snapshot (element descriptions + text) for replanning."""
+        return self._controller.see(window_title)
+
+    def catalog(self, max_level: PermissionLevel | None = None) -> str:
+        """Skill manifest text for the AI, hiding skills above ``max_level``."""
+        if max_level is None:
+            max_level = self._permissions.level
+        return REGISTRY.manifest(max_level=max_level)
+
+    @property
+    def controller(self) -> Any:
+        return self._controller
+
+    @property
+    def state_manager(self) -> StateManager:
+        return self._state

seedcode/computer/keyboard.py

@@ -0,0 +1,101 @@
+"""Keyboard driver: text typing and hotkeys via pyautogui.
+
+Typing uses a small per-key interval so target applications reliably receive
+every keystroke; hotkeys accept the pyautogui key-name vocabulary
+("ctrl", "alt", "shift", "win", "enter", "f5", single characters, ...).
+
+**Self-protection:** synthesized input goes to whatever window currently has
+focus. When a browser/app focus step silently failed, that window is often
+SeedCode's own terminal — so keystrokes meant for a web page would land in
+the user's prompt or, worse, an Alt+F4-style combo would close our console
+(the reported auto-exit bug). Two guards close both holes: window-closing
+combos are refused outright, and any input is refused while *our* window is
+focused (see :mod:`.selfguard`).
+"""
+
+from __future__ import annotations
+
+from . import selfguard
+
+# Bound one type_text call: pathological lengths point at a confused model.
+MAX_TEXT_LENGTH = 5_000
+
+# Keys that can dismiss or destroy a window. SeedCode never synthesizes
+# window-level close/dismiss keystrokes: closing an app is done through
+# windows.close_window (guarded), closing a browser tab through the browser
+# engine — never by pressing Alt+F4/Cmd+W blind at whatever has focus.
+_FORBIDDEN_COMBOS = (
+    {"alt", "f4"},      # close the focused window
+    {"cmd", "w"},       # macOS-style window close
+    {"ctrl", "shift", "w"},  # close all browser windows (not a plain tab)
+)
+
+# Keys allowed in hotkey combos (a safety vocabulary, not an exhaustive list
+# of what pyautogui supports — unknown names are rejected loudly).
+_MODIFIERS = {"ctrl", "alt", "shift", "win", "cmd", "fn"}
+_NAMED_KEYS = {
+    "enter", "return", "tab", "space", "backspace", "delete", "del", "esc",
+    "escape", "home", "end", "pageup", "pagedown", "up", "down", "left",
+    "right", "insert", "printscreen", "capslock", "numlock",
+} | {f"f{i}" for i in range(1, 25)}
+
+
+def _pyautogui():
+    import pyautogui
+
+    pyautogui.FAILSAFE = True
+    return pyautogui
+
+
+def validate_keys(keys: list[str]) -> list[str]:
+    """Normalise and validate hotkey names; raises ValueError on junk."""
+    cleaned = []
+    for key in keys:
+        name = str(key).strip().lower()
+        if not name:
+            continue
+        if name in _MODIFIERS or name in _NAMED_KEYS or len(name) == 1:
+            cleaned.append(name)
+        else:
+            raise ValueError(f"Unknown key '{key}' in hotkey combination.")
+    if not cleaned:
+        raise ValueError("Hotkey combination is empty.")
+    return cleaned
+
+
+def _assert_safe_input() -> None:
+    """Refuse synthesized input while SeedCode's own window has focus.
+
+    A keystroke aimed at another application whose focus step failed must
+    never be delivered into the agent's own prompt or console.
+    """
+    if selfguard.foreground_is_own():
+        raise ValueError(
+            "SeedCode's own window is focused; refusing to send keystrokes. "
+            "Focus the target application first (desktop focus_app / "
+            "focus_window)."
+        )
+
+
+def type_text(text: str, interval: float = 0.02) -> None:
+    _assert_safe_input()
+    if len(text) > MAX_TEXT_LENGTH:
+        raise ValueError(
+            f"Text is too long to type ({len(text)} chars; max {MAX_TEXT_LENGTH})."
+        )
+    _pyautogui().typewrite(text, interval=interval)
+
+
+def hotkey(keys: list[str]) -> None:
+    normalized = [k.strip().lower() for k in validate_keys(keys)]
+    combo = {k for k in normalized if k not in ("",)}
+    for forbidden in _FORBIDDEN_COMBOS:
+        if forbidden <= combo:
+            raise ValueError(
+                "Refusing to send a window-closing hotkey ("
+                + "+".join(sorted(forbidden))
+                + "). Close applications via the close_app skill or windows "
+                "driver instead — never with a blind keyboard shortcut."
+            )
+    _assert_safe_input()
+    _pyautogui().hotkey(*normalized)

seedcode/computer/logbook.py

@@ -0,0 +1,104 @@
+"""Structured execution log for the Computer Engine.
+
+Every run through the dispatcher records what actually happened — the skill
+chosen, the deterministic steps taken, whether verification passed, which
+recovery strategies were used, and the final status — as an ordered list of
+:class:`LogEntry` records. This is the engine's honest account of its own work:
+the dispatcher writes it, the AI reads a compact rendering of it, and nothing
+in it is fabricated. Success is only ever recorded when verification actually
+passed.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class Stage(str, Enum):
+    PLAN = "plan"            # a skill/action was selected
+    EXECUTE = "execute"     # deterministic steps ran
+    VERIFY = "verify"       # outcome checked
+    RECOVER = "recover"     # recovery strategies attempted
+    DONE = "done"           # final status
+    ERROR = "error"         # hard failure
+
+
+@dataclass(slots=True)
+class LogEntry:
+    stage: Stage
+    message: str
+    ok: bool = True
+
+    def render(self) -> str:
+        mark = "✓" if self.ok else "✗"
+        return f"{mark} [{self.stage.value}] {self.message}"
+
+
+@dataclass
+class ExecutionLog:
+    """An append-only trail of a single dispatched request."""
+
+    entries: list[LogEntry] = field(default_factory=list)
+
+    def add(self, stage: Stage, message: str, ok: bool = True) -> None:
+        self.entries.append(LogEntry(stage, message, ok))
+
+    def plan(self, message: str) -> None:
+        self.add(Stage.PLAN, message)
+
+    def execute(self, message: str, ok: bool = True) -> None:
+        self.add(Stage.EXECUTE, message, ok)
+
+    def verify(self, message: str, ok: bool = True) -> None:
+        self.add(Stage.VERIFY, message, ok)
+
+    def recover(self, message: str, ok: bool = True) -> None:
+        self.add(Stage.RECOVER, message, ok)
+
+    def done(self, message: str, ok: bool = True) -> None:
+        self.add(Stage.DONE if ok else Stage.ERROR, message, ok)
+
+    @property
+    def succeeded(self) -> bool:
+        """True only if a DONE (not ERROR) entry was recorded and nothing failed."""
+        return any(e.stage is Stage.DONE and e.ok for e in self.entries)
+
+    def render(self) -> str:
+        return "\n".join(e.render() for e in self.entries)
+
+    def summary(self) -> str:
+        """One-line result for the AI: the final DONE/ERROR message."""
+        for e in reversed(self.entries):
+            if e.stage in (Stage.DONE, Stage.ERROR):
+                return e.message
+        return self.entries[-1].message if self.entries else "nothing happened"
+
+    def persist(self, label: str = "") -> None:
+        """Append this run to ``~/.seedcode/logs/execution-<date>.jsonl``.
+
+        Best-effort: a full disk or locked file must never fail the action the
+        log describes. One JSON line per dispatched request keeps the file
+        greppable and machine-readable.
+        """
+        try:
+            from ..utils.helpers import app_dir
+
+            logs = app_dir() / "logs"
+            logs.mkdir(parents=True, exist_ok=True)
+            path = logs / time.strftime("execution-%Y%m%d.jsonl", time.localtime())
+            record = {
+                "ts": round(time.time(), 3),
+                "label": label,
+                "ok": self.succeeded,
+                "stages": [
+                    {"stage": e.stage.value, "ok": e.ok, "message": e.message}
+                    for e in self.entries
+                ],
+            }
+            with path.open("a", encoding="utf-8") as fh:
+                fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+        except Exception:
+            pass  # logging is evidence, not a dependency

seedcode/computer/mouse.py

@@ -0,0 +1,48 @@
+"""Mouse driver: move, click, drag, scroll via pyautogui.
+
+Coordinate validation lives in the controller (which knows the screen
+geometry); this module only performs the raw actions. ``FAILSAFE`` stays on:
+slamming the pointer into the top-left corner aborts any action — the user's
+emergency brake.
+"""
+
+from __future__ import annotations
+
+
+def _pyautogui():
+    import pyautogui
+
+    pyautogui.FAILSAFE = True
+    pyautogui.PAUSE = 0.05  # small settle between low-level actions
+    return pyautogui
+
+
+def position() -> tuple[int, int]:
+    """Current pointer position."""
+    point = _pyautogui().position()
+    return int(point.x), int(point.y)
+
+
+def move(x: int, y: int, duration: float = 0.2) -> None:
+    _pyautogui().moveTo(x, y, duration=duration)
+
+
+def click(x: int, y: int, button: str = "left", double: bool = False) -> None:
+    gui = _pyautogui()
+    clicks = 2 if double else 1
+    gui.click(x=x, y=y, clicks=clicks, button=button)
+
+
+def drag(x1: int, y1: int, x2: int, y2: int, duration: float = 0.5) -> None:
+    """Drag & drop: press at (x1, y1), release at (x2, y2)."""
+    gui = _pyautogui()
+    gui.moveTo(x1, y1, duration=0.2)
+    gui.dragTo(x2, y2, duration=max(0.2, duration), button="left")
+
+
+def scroll(amount: int, x: int | None = None, y: int | None = None) -> None:
+    """Scroll by ``amount`` notches (positive = up) at an optional position."""
+    gui = _pyautogui()
+    if x is not None and y is not None:
+        gui.moveTo(x, y, duration=0.1)
+    gui.scroll(amount)