seedcode-cli 6.1.5__py3-none-any.whl

seedcode/computer/controller.py

@@ -0,0 +1,324 @@
+"""ComputerController: the façade the desktop tools drive.
+
+Owns the driver modules (injectable for tests), validates every coordinate
+against the live virtual-desktop geometry (never a blind click), paces
+actions, and verifies each mutating action by reporting post-action state —
+the active window and the element under the pointer — so the agent loop can
+confirm outcomes instead of assuming them. Focus operations retry once
+before surfacing a failure.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from ..utils.logger import get_logger
+from . import browser as browser_driver
+from . import keyboard as keyboard_driver
+from . import mouse as mouse_driver
+from . import registry as registry_driver
+from . import screen as screen_driver
+from . import vision as vision_driver
+from . import windows as windows_driver
+
+_log = get_logger("computer")
+
+# One agent turn may not wait forever; desktop_wait is clamped to this.
+MAX_WAIT_S = 30.0
+# Settle time after mutating actions before state is read for verification.
+_VERIFY_DELAY_S = 0.3
+
+
+class ComputerError(Exception):
+    """A desktop action failed; the message is fed back to the model."""
+
+
+class ComputerController:
+    """High-level desktop actions with validation and built-in verification."""
+
+    def __init__(
+        self,
+        mouse: Any = mouse_driver,
+        keyboard: Any = keyboard_driver,
+        screen: Any = screen_driver,
+        windows: Any = windows_driver,
+        vision: Any = vision_driver,
+        registry: Any = registry_driver,
+        browser: Any = browser_driver,
+    ) -> None:
+        self.mouse = mouse
+        self.keyboard = keyboard
+        self.screen = screen
+        self.windows = windows
+        self.vision = vision
+        self.registry = registry
+        self.browser = browser
+        # Declare per-monitor-DPI-awareness v2 once, before any coordinate is
+        # read or acted on, so UIA geometry, screenshots, and pointer clicks
+        # all share one physical-pixel space (see :mod:`.dpi`). Best-effort and
+        # idempotent; a no-op off win32.
+        from . import dpi as _dpi
+
+        self.dpi = _dpi.ensure_dpi_awareness()
+
+    # --- coordinates ---------------------------------------------------------
+    def validate_point(self, x: Any, y: Any) -> tuple[int, int]:
+        """Coerce and bounds-check a coordinate pair against the desktop."""
+        try:
+            px, py = int(x), int(y)
+        except (TypeError, ValueError):
+            raise ComputerError(f"Coordinates must be integers, got ({x!r}, {y!r}).")
+        geo = self.screen.geometry()
+        if not geo.contains(px, py):
+            raise ComputerError(
+                f"({px}, {py}) is outside the desktop "
+                f"({geo.left}..{geo.left + geo.width - 1}, "
+                f"{geo.top}..{geo.top + geo.height - 1}). "
+                "Take a fresh desktop_see snapshot for current coordinates."
+            )
+        return px, py
+
+    # --- verification helpers ------------------------------------------------
+    def _state_after(self, x: int | None = None, y: int | None = None) -> str:
+        """Post-action state line appended to every mutating result."""
+        time.sleep(_VERIFY_DELAY_S)
+        active = self.windows.active_window()
+        parts = [f"active window: {active.describe() if active else '(none)'}"]
+        if x is not None and y is not None:
+            parts.append(f"element at ({x}, {y}): {self.vision.element_at(x, y)}")
+        return "[verify] " + "; ".join(parts)
+
+    # --- mouse ---------------------------------------------------------------
+    def mouse_move(self, x: Any, y: Any) -> str:
+        px, py = self.validate_point(x, y)
+        self.mouse.move(px, py)
+        return f"Moved pointer to ({px}, {py}).\n" + self._state_after(px, py)
+
+    def mouse_click(self, x: Any, y: Any, button: str = "left", double: bool = False) -> str:
+        px, py = self.validate_point(x, y)
+        if button not in ("left", "right"):
+            raise ComputerError("button must be 'left' or 'right'.")
+        self.mouse.click(px, py, button=button, double=double)
+        kind = "Double-clicked" if double else f"{button.capitalize()}-clicked"
+        return f"{kind} at ({px}, {py}).\n" + self._state_after(px, py)
+
+    def mouse_drag(self, x1: Any, y1: Any, x2: Any, y2: Any) -> str:
+        sx, sy = self.validate_point(x1, y1)
+        ex, ey = self.validate_point(x2, y2)
+        self.mouse.drag(sx, sy, ex, ey)
+        return f"Dragged ({sx}, {sy}) -> ({ex}, {ey}).\n" + self._state_after(ex, ey)
+
+    def mouse_scroll(self, amount: Any, x: Any = None, y: Any = None) -> str:
+        try:
+            notches = int(amount)
+        except (TypeError, ValueError):
+            raise ComputerError(f"Scroll amount must be an integer, got {amount!r}.")
+        notches = max(-50, min(50, notches))
+        point = None
+        if x is not None and y is not None:
+            point = self.validate_point(x, y)
+        self.mouse.scroll(notches, *(point or (None, None)))
+        where = f" at {point}" if point else ""
+        return f"Scrolled {notches} notches{where}.\n" + self._state_after(*(point or (None, None)))
+
+    # --- keyboard ------------------------------------------------------------
+    def type_text(self, text: str) -> str:
+        try:
+            self.keyboard.type_text(str(text))
+        except ValueError as exc:
+            raise ComputerError(str(exc))
+        shown = str(text) if len(str(text)) <= 60 else str(text)[:60] + "..."
+        return f"Typed {len(str(text))} chars ({shown!r}).\n" + self._state_after()
+
+    def hotkey(self, keys: list[str]) -> str:
+        try:
+            self.keyboard.hotkey([str(k) for k in keys])
+        except ValueError as exc:
+            raise ComputerError(str(exc))
+        return f"Pressed {'+'.join(str(k) for k in keys)}.\n" + self._state_after()
+
+    # --- windows -------------------------------------------------------------
+    def list_windows(self) -> str:
+        windows = self.windows.list_windows()
+        if not windows:
+            return "No open windows found."
+        return "Open windows:\n" + "\n".join(f"  - {w.describe()}" for w in windows)
+
+    def focus_window(self, title: str) -> str:
+        """Focus with one retry — Windows foreground rules are flaky."""
+        last_error: Exception | None = None
+        for attempt in (1, 2):
+            try:
+                self.windows.focus_window(title)
+                time.sleep(_VERIFY_DELAY_S)
+                active = self.windows.active_window()
+                if active and title.strip().lower() in active.title.lower():
+                    return f"Focused window.\n[verify] active window: {active.describe()}"
+                last_error = None
+            except Exception as exc:
+                last_error = exc
+            time.sleep(0.4)  # let the window manager settle, then retry
+        if last_error is not None:
+            raise ComputerError(f"Could not focus '{title}': {last_error}")
+        active = self.windows.active_window()
+        raise ComputerError(
+            f"Focus did not stick on '{title}'. Active window is "
+            f"{active.describe() if active else '(none)'} — check desktop_windows."
+        )
+
+    def open_app(self, target: str) -> str:
+        try:
+            message = self.windows.open_app(target)
+        except Exception as exc:
+            raise ComputerError(str(exc))
+        time.sleep(1.0)  # give the app a moment to show a window
+        return message + "\n" + self._state_after()
+
+    def close_app(self, title: str, force: bool = False) -> str:
+        try:
+            message = self.windows.close_window(title, force=force)
+        except Exception as exc:
+            raise ComputerError(str(exc))
+        return message + "\n" + self._state_after()
+
+    # --- screen & vision -----------------------------------------------------
+    def screenshot(
+        self,
+        region: tuple[int, int, int, int] | None = None,
+        monitor: int | None = None,
+        save_to=None,
+    ) -> str:
+        try:
+            path = self.screen.capture(region=region, monitor=monitor, save_to=save_to)
+        except Exception as exc:
+            raise ComputerError(f"Screenshot failed: {exc}")
+        return str(path)
+
+    def screen_info(self) -> str:
+        from . import dpi as _dpi
+
+        geo = self.screen.geometry()
+        lines = [
+            f"Virtual desktop: {geo.width}x{geo.height} at ({geo.left}, {geo.top})",
+            f"Monitors: {len(geo.monitors)}",
+        ]
+        for m in geo.monitors:
+            primary = " (primary)" if m.primary else ""
+            # Report each monitor's scale so the model understands why a
+            # 1920-wide panel may only be 1280 logical px, etc.
+            scale = _dpi.scale_for_point(m.left + m.width // 2, m.top + m.height // 2)
+            scale_txt = f" @ {int(scale * 100)}%" if scale != 1.0 else ""
+            lines.append(
+                f"  - Monitor {m.index}: {m.width}x{m.height} at "
+                f"({m.left}, {m.top}){scale_txt}{primary}"
+            )
+        lines.append(getattr(self, "dpi", _dpi.ensure_dpi_awareness()).describe())
+        return "\n".join(lines)
+
+    def see(self, window_title: str | None = None) -> str:
+        """UI-tree snapshot: what is on screen, with clickable coordinates."""
+        try:
+            title, elements = self.vision.snapshot(window_title)
+        except ValueError as exc:
+            raise ComputerError(str(exc))
+        except Exception as exc:
+            raise ComputerError(f"Could not read the UI tree: {exc}")
+        return self.vision.describe_snapshot(title, elements)
+
+    def wait(self, seconds: Any) -> str:
+        try:
+            duration = float(seconds)
+        except (TypeError, ValueError):
+            raise ComputerError(f"Wait time must be a number, got {seconds!r}.")
+        duration = max(0.0, min(duration, MAX_WAIT_S))
+        time.sleep(duration)
+        return f"Waited {duration:g}s.\n" + self._state_after()
+
+    # --- registry ------------------------------------------------------------
+    def registry_read(self, path: str, name: str = "") -> str:
+        try:
+            return self.registry.read_value(path, name)
+        except FileNotFoundError:
+            raise ComputerError(f"Registry key or value not found: {path} : {name}")
+        except (ValueError, OSError, RuntimeError) as exc:
+            raise ComputerError(f"Registry read failed: {exc}")
+
+    def registry_list(self, path: str) -> str:
+        try:
+            return self.registry.list_key(path)
+        except FileNotFoundError:
+            raise ComputerError(f"Registry key not found: {path}")
+        except (ValueError, OSError, RuntimeError) as exc:
+            raise ComputerError(f"Registry list failed: {exc}")
+
+    def registry_write(self, path: str, name: str, value: str, value_type: str) -> str:
+        try:
+            return self.registry.write_value(path, name, value, value_type)
+        except (ValueError, OSError, RuntimeError) as exc:
+            raise ComputerError(f"Registry write failed: {exc}")
+
+    def registry_delete(self, path: str, name: str) -> str:
+        try:
+            return self.registry.delete_value(path, name)
+        except FileNotFoundError:
+            raise ComputerError(f"Registry key or value not found: {path} : {name}")
+        except (ValueError, OSError, RuntimeError) as exc:
+            raise ComputerError(f"Registry delete failed: {exc}")
+
+    # --- browser -------------------------------------------------------------
+    def browser_navigate(self, url: str) -> str:
+        """Open a URL in the user's default browser."""
+        try:
+            return self.browser.navigate(url)
+        except Exception as exc:
+            raise ComputerError(f"Browser navigation failed: {exc}")
+
+    def browser_search(self, query: str, engine: str = "google") -> str:
+        """Run a web search in the user's default browser."""
+        try:
+            return self.browser.search(query, engine=engine)
+        except Exception as exc:
+            raise ComputerError(f"Browser search failed: {exc}")
+
+    def browser_default(self) -> str:
+        """Report which browser is the system default."""
+        try:
+            return self.browser.default_browser().describe()
+        except Exception as exc:
+            raise ComputerError(f"Could not determine the default browser: {exc}")
+
+    def browser_click(self, selector: str, selector_type: str = "css") -> str:
+        """Click element in browser."""
+        try:
+            return self.browser.click_element(selector, selector_type)
+        except Exception as exc:
+            raise ComputerError(f"Browser click failed: {exc}")
+
+    def browser_type(self, selector: str, text: str, selector_type: str = "css") -> str:
+        """Type into browser element."""
+        try:
+            return self.browser.type_in_element(selector, text, selector_type)
+        except Exception as exc:
+            raise ComputerError(f"Browser typing failed: {exc}")
+
+    def browser_info(self) -> str:
+        """Get current browser page info."""
+        try:
+            return self.browser.get_page_info()
+        except Exception as exc:
+            raise ComputerError(f"Browser info failed: {exc}")
+
+    def browser_find(self, selector: str, selector_type: str = "css") -> str:
+        """Find elements in browser."""
+        try:
+            return self.browser.find_elements(selector, selector_type)
+        except Exception as exc:
+            raise ComputerError(f"Browser find failed: {exc}")
+
+    def browser_close(self) -> str:
+        """Close browser."""
+        try:
+            return self.browser.close_browser()
+        except Exception as exc:
+            raise ComputerError(f"Browser close failed: {exc}")

seedcode/computer/dispatcher.py

@@ -0,0 +1,272 @@
+"""Skill dispatcher: the coordinator between AI decisions and deterministic work.
+
+The dispatcher is the single choke point through which every AI-selected action
+flows. It:
+
+1. looks up the skill (or builds an ad-hoc action for a semantic UI verb),
+2. executes it deterministically via the :class:`SkillContext`,
+3. verifies the declared outcome with the :class:`VerificationEngine`,
+4. on failure, runs the :class:`RecoveryEngine`'s local strategy ladder,
+5. records every stage in an :class:`ExecutionLog`,
+6. returns a :class:`DispatchResult` — verified success, or a failure carrying
+   a replan hint the AI planner uses (and only then) to think again.
+
+The AI never sees the inner steps or any coordinates. It selects a skill by
+name or issues a semantic UI verb with element *descriptions*; the dispatcher
+and the engines below it do the rest.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from .logbook import ExecutionLog
+from .recovery import RecoveryEngine
+from .skills import Outcome, Skill, SkillContext, SkillError, SkillRegistry
+from .verifier import VerificationEngine
+
+
+@dataclass(slots=True)
+class DispatchResult:
+    ok: bool
+    detail: str
+    log: ExecutionLog
+    replan_hint: str | None = None
+
+    def for_model(self) -> str:
+        """Concise text the AI reads back — verified evidence, never raw steps."""
+        head = self.detail
+        if not self.ok and self.replan_hint:
+            return f"{head}\nReplan hint: {self.replan_hint}"
+        return head
+
+
+# Semantic UI verbs the AI may issue directly (no named skill). Each maps to a
+# deterministic body over the resolver + controller; every target is a
+# *description*, resolved to coordinates internally.
+class _SemanticActions:
+    """Ad-hoc, description-driven UI actions built on the resolver."""
+
+    @staticmethod
+    def click(ctx: SkillContext, params: dict[str, Any], *, button="left", double=False) -> Outcome:
+        target = str(params.get("target", "")).strip()
+        if not target:
+            raise SkillError("ui action requires a 'target' description")
+        detail = ctx.click_described(target, button=button, double=double)
+        return Outcome(detail, {"element": target})
+
+    @staticmethod
+    def type(ctx: SkillContext, params: dict[str, Any]) -> Outcome:
+        target = str(params.get("target", "")).strip()
+        text = str(params.get("text", ""))
+        secret = bool(params.get("secret", False))
+        if not target:
+            raise SkillError("ui_type requires a 'target' description")
+        detail = ctx.type_into(target, text, secret=secret)
+        return Outcome(detail)
+
+    @staticmethod
+    def wait_for(ctx: SkillContext, params: dict[str, Any]) -> Outcome:
+        target = str(params.get("target") or params.get("description", "")).strip()
+        if not target:
+            raise SkillError("ui_wait_for requires a 'target' description")
+        return Outcome(f"waiting for {target}", {"element": target})
+
+    @staticmethod
+    def assert_(ctx: SkillContext, params: dict[str, Any]) -> Outcome:
+        target = str(params.get("target") or params.get("description", "")).strip()
+        if not target:
+            raise SkillError("ui_assert requires a 'target' description")
+        return Outcome(f"asserting {target}", {"element": target})
+
+
+_SEMANTIC = {
+    "ui_click": lambda c, p: _SemanticActions.click(c, p),
+    "ui_double_click": lambda c, p: _SemanticActions.click(c, p, double=True),
+    "ui_right_click": lambda c, p: _SemanticActions.click(c, p, button="right"),
+    "ui_type": lambda c, p: _SemanticActions.type(c, p),
+    "ui_wait_for": lambda c, p: _SemanticActions.wait_for(c, p),
+    "ui_assert": lambda c, p: _SemanticActions.assert_(c, p),
+}
+
+# Semantic verbs that actually manipulate the UI. Observation verbs
+# (ui_wait_for, ui_assert) are harmless on a browser and stay allowed.
+_MUTATING_SEMANTIC = ("ui_click", "ui_double_click", "ui_right_click", "ui_type")
+
+# Window-title fragments that identify a browser. Titles end with the browser
+# name on every mainstream platform ("YouTube — Google Chrome").
+_BROWSER_MARKERS = (
+    "google chrome", "chrome", "microsoft edge", "edge", "mozilla firefox",
+    "firefox", "brave", "opera", "vivaldi", "chromium", "safari",
+)
+
+
+def _is_browser_window(title: str | None) -> bool:
+    """Whether a window title belongs to a web browser."""
+    low = (title or "").strip().lower()
+    if not low:
+        return False
+    return any(marker in low for marker in _BROWSER_MARKERS)
+
+
+def _browser_skill_names(registry: SkillRegistry) -> str:
+    """The browser skills to point the planner at, from the live registry."""
+    wanted = (
+        "youtube_play", "youtube_search", "google_search", "web_search",
+        "open_url", "new_tab", "close_tab", "switch_tab", "back", "forward",
+        "refresh",
+    )
+    available = [name for name in wanted if registry.get(name) is not None]
+    return ", ".join(available)
+
+
+class SkillDispatcher:
+    """Coordinates execute → verify → recover for one action at a time."""
+
+    def __init__(
+        self,
+        controller: Any,
+        resolver: Any,
+        state: Any,
+        permissions: Any,
+        registry: SkillRegistry,
+        verifier: VerificationEngine | None = None,
+        recovery: RecoveryEngine | None = None,
+    ) -> None:
+        self._ctx = SkillContext(
+            controller=controller, resolver=resolver, state=state, permissions=permissions
+        )
+        self._registry = registry
+        self._verifier = verifier or VerificationEngine(
+            vision=getattr(controller, "vision", None),
+            windows=getattr(controller, "windows", None),
+        )
+        self._recovery = recovery or RecoveryEngine(controller=controller, state=state)
+
+    def _refuse_browser_ui(self, verb: str, params: dict[str, Any]) -> str | None:
+        """The refusal message for a ui_* verb on a browser, or None to allow.
+
+        Deliberately fails *open*: if the foreground window cannot be read, the
+        action proceeds. A guard that blocked on uncertainty would break native
+        automation on any machine whose window driver is unavailable.
+        """
+        controller = getattr(self._ctx, "controller", None)
+        windows = getattr(controller, "windows", None)
+        if windows is None:
+            return None
+        try:
+            active = windows.active_window()
+        except Exception:
+            return None
+        title = getattr(active, "title", None)
+        if not _is_browser_window(title):
+            return None
+        target = str(params.get("target", "")).strip()
+        skills = _browser_skill_names(self._registry)
+        return (
+            f"{verb} is not available on browser windows"
+            f'{f" (target: {target!r})" if target else ""}. '
+            "Browser work is handled by complete skills that navigate, dismiss "
+            "popups, select results, and verify the outcome in one step — "
+            "clicking through a page yourself is slower and breaks whenever the "
+            f"site changes. Use computer_run with one of: {skills}. "
+            "For example, to play a song: "
+            'computer_run(skill="youtube_play", params={"query": "<song name>"}).'
+        )
+
+    def dispatch(
+        self, name: str, params: dict[str, Any] | None = None, expected: dict[str, Any] | None = None
+    ) -> DispatchResult:
+        """Run one skill or semantic UI verb end-to-end with verification.
+
+        Every outcome — success, failure, and everything the engine did on the
+        way — is appended to the structured execution log on disk.
+        """
+        result = self._dispatch(name, params, expected)
+        result.log.persist(label=str(name).strip().lower())
+        return result
+
+    def _dispatch(
+        self, name: str, params: dict[str, Any] | None = None, expected: dict[str, Any] | None = None
+    ) -> DispatchResult:
+        log = ExecutionLog()
+        params = params or {}
+        name = str(name).strip().lower()
+
+        skill = self._registry.get(name)
+        semantic = _SEMANTIC.get(name)
+        if skill is None and semantic is None:
+            log.done(f"unknown skill or action '{name}'", ok=False)
+            return DispatchResult(
+                False, f"No such skill '{name}'.", log,
+                replan_hint="Choose a skill from the catalog or a ui_* action.",
+            )
+
+        label = skill.name if skill else name
+        log.plan(f"selected {label} {params}")
+
+        # --- browser guard ---------------------------------------------------
+        # Browser workflows are owned end-to-end by the browser skills. Letting
+        # the AI click its way around a web page is what this architecture
+        # exists to prevent: it burns turns, breaks on every layout change, and
+        # drags OCR into a problem a URL already solves. So a mutating ui_*
+        # verb aimed at a browser window is refused with a pointer to the skill
+        # that does the whole job.
+        if name in _MUTATING_SEMANTIC:
+            refusal = self._refuse_browser_ui(name, params)
+            if refusal is not None:
+                log.done(refusal, ok=False)
+                return DispatchResult(False, refusal, log, replan_hint=refusal)
+
+        # --- execute ---------------------------------------------------------
+        def _run() -> Outcome:
+            if skill is not None:
+                return skill.run(self._ctx, params)
+            return semantic(self._ctx, params)
+
+        try:
+            outcome = _run()
+        except SkillError as exc:
+            log.execute(str(exc), ok=False)
+            log.done(f"{label} failed: {exc}", ok=False)
+            return DispatchResult(False, str(exc), log, replan_hint=str(exc))
+        except Exception as exc:  # permission denial, driver error, ...
+            log.execute(f"{type(exc).__name__}: {exc}", ok=False)
+            log.done(f"{label} failed: {exc}", ok=False)
+            return DispatchResult(
+                False, f"{label} could not run: {exc}", log,
+                replan_hint=f"{type(exc).__name__}: {exc}",
+            )
+        log.execute(outcome.detail)
+
+        # Caller-supplied expectation overrides the skill's own.
+        want = expected or outcome.expected
+
+        # --- verify ----------------------------------------------------------
+        result = self._verifier.verify(want)
+        if result.ok:
+            log.verify(result.detail)
+            log.done(outcome.detail)
+            return DispatchResult(True, outcome.detail, log)
+        log.verify(result.detail, ok=False)
+
+        # --- recover ---------------------------------------------------------
+        recovery = self._recovery.recover(
+            action=lambda: _run().detail,
+            verify=lambda: self._verifier.verify(want),
+            window_title=outcome.window_title,
+            app_target=outcome.app_target,
+        )
+        if recovery.recovered:
+            log.recover(f"{recovery.detail} (tried: {', '.join(recovery.strategies_tried)})")
+            log.done(outcome.detail)
+            return DispatchResult(True, outcome.detail, log)
+
+        log.recover(recovery.detail, ok=False)
+        hint = (
+            f"'{label}' executed but verification failed ({result.detail}); local "
+            f"recovery ({', '.join(recovery.strategies_tried) or 'none'}) did not help."
+        )
+        log.done(hint, ok=False)
+        return DispatchResult(False, hint, log, replan_hint=hint)