seedcode-cli 6.1.5__py3-none-any.whl

seedcode/computer/semantic.py

@@ -0,0 +1,100 @@
+"""Semantic element actions: id in, verified action out.
+
+The bridge between the Screen Intelligence Engine (:mod:`.screen_state`) and
+the low-level drivers. The AI never supplies coordinates here; it supplies a
+stable element id from a previous query, and this module:
+
+1. resolves the id through the engine (cache → fresh validation),
+2. re-locates the element if the UI changed (bounded stale-requery),
+3. performs the action through the controller's guarded drivers,
+4. returns a compact, model-readable result with post-action state.
+
+Failures raise structured operator errors (ELEMENT_NOT_FOUND,
+STALE_ELEMENT) that flow back to the planner like any other tool result.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..core.errors import ElementNotFoundError, StaleElementError
+from ..core.limits import MAX_STALE_REQUERY
+from .screen_state import Element, ScreenEngine, get_screen_engine
+
+
+class SemanticActions:
+    """Element-id-addressed actions over the screen engine + controller."""
+
+    def __init__(self, engine: ScreenEngine | None = None, controller: Any = None) -> None:
+        self._engine = engine
+        self._controller = controller
+
+    # --- plumbing -------------------------------------------------------------
+    @property
+    def engine(self) -> ScreenEngine:
+        if self._engine is None:
+            self._engine = get_screen_engine()
+        return self._engine
+
+    @property
+    def controller(self) -> Any:
+        if self._controller is None:
+            from .controller import ComputerController
+
+            self._controller = ComputerController()
+        return self._controller
+
+    def _resolve(self, element_id: str) -> Element:
+        """Resolve an id to a fresh element with bounded stale-requery.
+
+        A stale id (UI changed between query and action) triggers up to
+        ``MAX_STALE_REQUERY`` re-locate cycles by the element's identity
+        (role + name + automation id) before giving up with STALE_ELEMENT.
+        """
+        try:
+            return self.engine.get_element(element_id)
+        except StaleElementError as exc:
+            cached = self.engine._by_id.get((element_id or "").strip().lower())
+            identity = getattr(cached, "name", "") or element_id
+            for _ in range(MAX_STALE_REQUERY):
+                try:
+                    return self.engine.find_element(identity, fresh=True)
+                except ElementNotFoundError:
+                    continue
+            raise exc
+
+    # --- actions ---------------------------------------------------------------
+    def click(self, element_id: str, *, button: str = "left", double: bool = False) -> str:
+        el = self._resolve(element_id)
+        self.controller.mouse_click(el.x, el.y, button=button, double=double)
+        return f"clicked {el.describe()}"
+
+    def type_into(self, element_id: str, text: str, *, secret: bool = False) -> str:
+        el = self._resolve(element_id)
+        self.controller.mouse_click(el.x, el.y)  # focus first
+        self.controller.type_text(text)
+        shown = "•" * len(text) if secret else text
+        return f"typed '{shown}' into {el.describe()}"
+
+    def focus(self, element_id: str) -> str:
+        el = self._resolve(element_id)
+        self.controller.mouse_click(el.x, el.y)
+        return f"focused {el.describe()}"
+
+    def press(self, element_id: str, keys: list[str]) -> str:
+        """Focus the element, then send a (guarded) hotkey to it."""
+        el = self._resolve(element_id)
+        self.controller.mouse_click(el.x, el.y)
+        self.controller.hotkey([str(k) for k in keys])
+        return f"pressed {'+'.join(keys)} on {el.describe()}"
+
+    def select(self, element_id: str) -> str:
+        """Select a combobox/list option by clicking it (checkbox/radio included)."""
+        return self.click(element_id)
+
+    def check(self, element_id: str, *, checked: bool = True) -> str:
+        el = self._resolve(element_id)
+        if el.checked is checked:
+            return f"{el.describe()} already {'checked' if checked else 'unchecked'}"
+        self.controller.mouse_click(el.x, el.y)
+        return f"{'checked' if checked else 'unchecked'} {el.describe()}"

seedcode/computer/skills.py

@@ -0,0 +1,139 @@
+"""Skill engine: high-level, deterministic procedures the AI selects by name.
+
+A *skill* is Seed Code's unit of knowledge — "launch an app", "search
+YouTube", "create a Python project". The AI planner chooses a skill and
+supplies parameters; the skill expands into a fixed sequence of deterministic
+Computer Engine operations. The AI never sees those inner steps and never
+produces coordinates, keystrokes, or click sequences.
+
+Each skill declares:
+
+* ``name`` / ``summary`` / ``params`` — its catalog entry (what the AI reads).
+* ``level`` — the :class:`PermissionLevel` required to run it.
+* ``run(ctx, params) -> Outcome`` — the deterministic body, given a
+  :class:`SkillContext` that exposes the controller, element resolver, state
+  manager and permission manager.
+
+Skills raise :class:`SkillError` on unrecoverable failure; they return an
+:class:`Outcome` carrying the expectation the dispatcher will verify. Skills
+contain no AI calls and work offline.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from ..tools.permissions import PermissionLevel
+
+
+class SkillError(Exception):
+    """A skill could not complete (after its own inline handling)."""
+
+
+@dataclass(slots=True)
+class Outcome:
+    """What a skill produced, plus how to verify it succeeded."""
+
+    detail: str
+    expected: dict[str, Any] | None = None
+    # Hints the dispatcher passes to recovery if verification fails.
+    window_title: str | None = None
+    app_target: str | None = None
+
+
+@dataclass
+class SkillContext:
+    """Everything a skill body may use — all deterministic, no AI."""
+
+    controller: Any            # ComputerController: mouse/keyboard/windows/...
+    resolver: Any              # ElementResolver: description -> point
+    state: Any                 # StateManager: engine memory
+    permissions: Any           # PermissionManager: require()/confirm_action()
+
+    def click_described(self, description: str, *, button: str = "left", double: bool = False) -> str:
+        """Resolve a described element and click it — the resolver owns coords."""
+        hit = self.resolver.resolve(description)
+        self.controller.mouse_click(hit.x, hit.y, button, double)
+        self.state.record_action(f"clicked {description}")
+        return f"clicked {hit.describe()}"
+
+    def type_into(self, description: str, text: str, *, secret: bool = False) -> str:
+        """Focus a described field, then type into it."""
+        hit = self.resolver.resolve(description)
+        self.controller.mouse_click(hit.x, hit.y)
+        self.controller.type_text(text)
+        self.state.record_action(f"typed into {description}")
+        shown = "•" * len(text) if secret else text
+        return f"typed '{shown}' into {hit.describe()}"
+
+
+# A skill body: (ctx, params) -> Outcome.
+SkillBody = Callable[[SkillContext, dict[str, Any]], Outcome]
+
+
+@dataclass(slots=True)
+class Skill:
+    """A named, permissioned, deterministic procedure."""
+
+    name: str
+    summary: str
+    level: PermissionLevel
+    body: SkillBody
+    params: dict[str, str] = field(default_factory=dict)
+    sensitive: bool = False  # requires per-action confirmation even if level ok
+
+    def run(self, ctx: SkillContext, params: dict[str, Any]) -> Outcome:
+        # Enforce the skill's permission floor before doing anything.
+        ctx.permissions.require(self.level, f"skill '{self.name}'")
+        return self.body(ctx, params or {})
+
+
+class SkillRegistry:
+    """The catalog of known skills, keyed by name."""
+
+    def __init__(self) -> None:
+        self._skills: dict[str, Skill] = {}
+
+    def register(self, skill: Skill) -> Skill:
+        self._skills[skill.name] = skill
+        return skill
+
+    def get(self, name: str) -> Skill | None:
+        return self._skills.get(str(name).strip().lower())
+
+    def all(self) -> list[Skill]:
+        return sorted(self._skills.values(), key=lambda s: s.name)
+
+    def manifest(self, max_level: PermissionLevel | None = None) -> str:
+        """Human/AI-readable catalog, optionally hiding skills above a level."""
+        lines = []
+        for skill in self.all():
+            if max_level is not None and skill.level > max_level:
+                continue
+            args = ", ".join(skill.params) if skill.params else "no params"
+            lines.append(f"- {skill.name}({args}) — {skill.summary}")
+        return "\n".join(lines)
+
+
+# Module-level catalog populated by ``catalog.py`` at import time.
+REGISTRY = SkillRegistry()
+
+
+def skill(name, summary, level, params=None, sensitive=False):
+    """Decorator: register a function as a skill body."""
+
+    def deco(fn: SkillBody) -> SkillBody:
+        REGISTRY.register(
+            Skill(
+                name=name.strip().lower(),
+                summary=summary,
+                level=level,
+                body=fn,
+                params=params or {},
+                sensitive=sensitive,
+            )
+        )
+        return fn
+
+    return deco

seedcode/computer/state.py

@@ -0,0 +1,199 @@
+"""Computer Engine execution state.
+
+The engine keeps its own memory of the machine so the AI never has to re-ask
+for it: what app is focused, where the pointer is, what's on the clipboard,
+which directory the terminal is in, and a short trail of the last actions. The
+:class:`StateManager` refreshes the live parts (focused window, pointer) on
+demand from the drivers and records the rest as the dispatcher executes skills.
+
+This module is deterministic and offline — it only reads driver state and holds
+values; it never talks to an AI provider.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+# Keep the action trail short; it exists to give the AI recent context, not a
+# full audit log (that's the logbook's job).
+_MAX_TRAIL = 12
+
+
+@dataclass
+class ComputerState:
+    """A snapshot of what the Computer Engine believes about the machine."""
+
+    focused_window: str | None = None
+    focused_app: str | None = None
+    pointer: tuple[int, int] | None = None
+    clipboard_preview: str | None = None
+    terminal_cwd: str | None = None
+    current_project: str | None = None
+    open_browser_url: str | None = None
+    modifiers_held: tuple[str, ...] = ()
+    # Titles of the apps currently holding a visible window, refreshed from the
+    # window driver so the planner knows what is already open (and the engine
+    # can focus instead of launching a duplicate).
+    running_apps: tuple[str, ...] = ()
+    # The high-level task the engine is currently executing, set by the engine
+    # when a skill is dispatched — gives the AI continuity across steps.
+    current_task: str | None = None
+    recent_actions: list[str] = field(default_factory=list)
+
+    @property
+    def previous_action(self) -> str | None:
+        """The last action the engine performed, if any."""
+        return self.recent_actions[-1] if self.recent_actions else None
+
+    def describe(self) -> str:
+        """Compact, model-facing rendering of the current state."""
+        rows: list[tuple[str, str]] = []
+        if self.current_task:
+            rows.append(("task", self.current_task))
+        if self.focused_app:
+            rows.append(("app", self.focused_app))
+        if self.focused_window:
+            rows.append(("window", self.focused_window))
+        if self.pointer is not None:
+            rows.append(("pointer", f"({self.pointer[0]}, {self.pointer[1]})"))
+        if self.clipboard_preview:
+            rows.append(("clipboard", self.clipboard_preview))
+        if self.terminal_cwd:
+            rows.append(("terminal", self.terminal_cwd))
+        if self.current_project:
+            rows.append(("project", self.current_project))
+        if self.open_browser_url:
+            rows.append(("browser", self.open_browser_url))
+        if self.modifiers_held:
+            rows.append(("modifiers", "+".join(self.modifiers_held)))
+        if self.running_apps:
+            rows.append(("open apps", ", ".join(self.running_apps[:8])))
+        if not rows and not self.recent_actions:
+            return "Computer state: (nothing observed yet)"
+        lines = ["Computer state:"]
+        lines += [f"  {label}: {value}" for label, value in rows]
+        if self.recent_actions:
+            lines.append("  recent:")
+            lines += [f"    - {a}" for a in self.recent_actions[-5:]]
+        return "\n".join(lines)
+
+
+class StateManager:
+    """Owns the engine's :class:`ComputerState` and keeps it fresh.
+
+    The live bits (focused window, pointer position) are pulled from the driver
+    modules on :meth:`refresh`; everything else is recorded by the dispatcher as
+    skills run so the AI gets continuity across steps for free.
+    """
+
+    def __init__(self, windows: Any = None, mouse: Any = None) -> None:
+        # Drivers are injectable so tests can drive the manager without a real
+        # desktop; lazy-imported here to keep import order clean off-win32.
+        if windows is None:
+            from . import windows as windows  # type: ignore
+        if mouse is None:
+            from . import mouse as mouse  # type: ignore
+        self._windows = windows
+        self._mouse = mouse
+        self.state = ComputerState()
+
+    def refresh(self) -> ComputerState:
+        """Re-read the live parts of the state from the drivers.
+
+        Best-effort: a driver that raises (no desktop, flaky COM) leaves the
+        previous value in place rather than crashing the turn.
+        """
+        try:
+            active = self._windows.active_window()
+            if active is not None:
+                self.state.focused_window = active.title or None
+                self.state.focused_app = _app_from_title(active.title)
+        except Exception:
+            pass
+        try:
+            pos = self._mouse.position()
+            if pos is not None:
+                self.state.pointer = (int(pos[0]), int(pos[1]))
+        except Exception:
+            pass
+        try:
+            # Which apps already have a window: lets the planner focus an open
+            # app instead of launching a second copy.
+            apps: list[str] = []
+            for win in self._windows.list_windows():
+                app = _app_from_title(getattr(win, "title", None))
+                if app and app not in apps:
+                    apps.append(app)
+            if apps:
+                self.state.running_apps = tuple(apps)
+        except Exception:
+            pass
+        try:
+            url = self._browser_url()
+            if url:
+                self.state.open_browser_url = url
+        except Exception:
+            pass
+        return self.state
+
+    def _browser_url(self) -> str | None:
+        """The last URL the default browser was sent to (best-effort)."""
+        try:
+            from . import browser as browser_driver
+
+            return browser_driver.current_url()
+        except Exception:
+            return None
+
+    # --- recording (called by the dispatcher / skills) ----------------------
+    def record_action(self, description: str) -> None:
+        trail = self.state.recent_actions
+        trail.append(description)
+        if len(trail) > _MAX_TRAIL:
+            del trail[: len(trail) - _MAX_TRAIL]
+
+    def set_clipboard(self, text: str | None) -> None:
+        if not text:
+            self.state.clipboard_preview = None
+        else:
+            preview = text.strip().replace("\n", " ")
+            self.state.clipboard_preview = preview[:60] + ("…" if len(preview) > 60 else "")
+
+    def set_terminal_cwd(self, cwd: str | None) -> None:
+        self.state.terminal_cwd = cwd
+
+    def set_project(self, project: str | None) -> None:
+        self.state.current_project = project
+
+    def set_browser_url(self, url: str | None) -> None:
+        self.state.open_browser_url = url
+
+    def set_task(self, task: str | None) -> None:
+        """Record the high-level task the engine is currently executing."""
+        self.state.current_task = task
+
+    def set_modifiers(self, modifiers: "tuple[str, ...] | list[str]") -> None:
+        """Record which modifier keys the engine is currently holding down."""
+        self.state.modifiers_held = tuple(str(m).lower() for m in modifiers)
+
+    def note_focus(self, window_title: str | None) -> None:
+        """Record a focus change the driver just performed."""
+        if window_title:
+            self.state.focused_window = window_title
+            self.state.focused_app = _app_from_title(window_title)
+
+
+def _app_from_title(title: str | None) -> str | None:
+    """Best-effort app name from a window title.
+
+    Windows titles are usually ``Document — App`` or ``Document - App``; the
+    trailing segment is the app. Falls back to the whole title.
+    """
+    if not title:
+        return None
+    for sep in (" — ", " - ", " – "):
+        if sep in title:
+            return title.rsplit(sep, 1)[-1].strip() or title.strip()
+    return title.strip()

seedcode/computer/verifier.py

@@ -0,0 +1,177 @@
+"""Verification engine: never trust an action without checking its result.
+
+After the Computer Engine performs an action, the verifier confirms the world
+actually changed the way the skill expected — a window appeared, an element is
+present, text is on screen, a file exists, a process is running. Verification
+is deterministic and offline; it reads driver/OS state and returns a
+:class:`VerifyResult` the dispatcher uses to decide success vs. recovery.
+
+Expectations are plain data (a dict the skill or the AI supplies as
+``expected``), so the AI can state *what* success looks like without ever
+touching *how* it is checked.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(slots=True)
+class VerifyResult:
+    """Outcome of a verification check."""
+
+    ok: bool
+    detail: str
+
+    def __bool__(self) -> bool:  # lets callers write ``if verify(...):``
+        return self.ok
+
+
+# Supported expectation kinds. An ``expected`` dict names one of these plus its
+# argument, e.g. {"window": "Notepad"} or {"file_exists": "C:/x/out.txt"}.
+_KINDS = (
+    "window", "window_gone", "element", "text", "file_exists",
+    "folder_exists", "process", "browser_url",
+)
+
+
+class VerificationEngine:
+    """Checks expected outcomes against live machine state."""
+
+    # How long to keep re-checking a not-yet-true expectation before failing;
+    # UI and windows settle asynchronously, so a single check is too eager.
+    def __init__(
+        self,
+        vision: Any = None,
+        windows: Any = None,
+        browser: Any = None,
+        timeout_s: float = 4.0,
+        poll_s: float = 0.3,
+    ) -> None:
+        if vision is None:
+            from . import vision as vision  # type: ignore
+        if windows is None:
+            from . import windows as windows  # type: ignore
+        self._vision = vision
+        self._windows = windows
+        # Browser is optional (selenium may be absent); resolve lazily on use.
+        self._browser = browser
+        self._timeout_s = timeout_s
+        self._poll_s = poll_s
+
+    def verify(self, expected: dict[str, Any] | None) -> VerifyResult:
+        """Check an ``expected`` outcome dict; no expectation ⇒ pass.
+
+        Polls until the expectation holds or the timeout elapses, so a window
+        or element that appears a moment after the action still verifies.
+        """
+        if not expected:
+            return VerifyResult(True, "no expectation to verify")
+        kind = next((k for k in _KINDS if k in expected), None)
+        if kind is None:
+            return VerifyResult(True, f"unknown expectation {list(expected)}; skipped")
+        arg = expected[kind]
+
+        deadline = time.monotonic() + self._timeout_s
+        last = ""
+        while True:
+            ok, last = self._check(kind, arg)
+            if ok:
+                return VerifyResult(True, last)
+            if time.monotonic() >= deadline:
+                return VerifyResult(False, last)
+            time.sleep(self._poll_s)
+
+    # --- individual checks ---------------------------------------------------
+    def _check(self, kind: str, arg: Any) -> tuple[bool, str]:
+        try:
+            handler = getattr(self, f"_check_{kind}")
+        except AttributeError:
+            return True, f"no checker for {kind}"
+        try:
+            return handler(arg)
+        except Exception as exc:
+            return False, f"{kind} check errored: {exc}"
+
+    def _check_window(self, title: str) -> tuple[bool, str]:
+        want = str(title).strip().lower()
+        for w in self._windows.list_windows():
+            if want in w.title.lower():
+                return True, f'window "{w.title}" present'
+        return False, f'no window matching "{title}"'
+
+    def _check_window_gone(self, title: str) -> tuple[bool, str]:
+        want = str(title).strip().lower()
+        present = [w for w in self._windows.list_windows() if want in w.title.lower()]
+        if present:
+            return False, f'window "{title}" still open'
+        return True, f'window "{title}" closed'
+
+    def _check_element(self, description: str) -> tuple[bool, str]:
+        from .resolver import ElementResolver, ResolveError
+
+        try:
+            hit = ElementResolver(vision=self._vision).resolve(str(description))
+            return True, f"element present: {hit.describe()}"
+        except ResolveError as exc:
+            return False, str(exc)
+
+    def _check_text(self, text: str) -> tuple[bool, str]:
+        want = str(text).strip().lower()
+        try:
+            _title, elements = self._vision.snapshot()
+        except Exception:
+            elements = []
+        for el in elements:
+            if want in (el.name or "").lower():
+                return True, f'text "{text}" visible'
+        return False, f'text "{text}" not found on screen'
+
+    def _check_file_exists(self, path: str) -> tuple[bool, str]:
+        exists = os.path.isfile(str(path))
+        return exists, f'file {"exists" if exists else "missing"}: {path}'
+
+    def _check_folder_exists(self, path: str) -> tuple[bool, str]:
+        exists = os.path.isdir(str(path))
+        return exists, f'folder {"exists" if exists else "missing"}: {path}'
+
+    def _check_process(self, name: str) -> tuple[bool, str]:
+        want = str(name).strip().lower()
+        try:
+            import psutil  # type: ignore
+
+            for proc in psutil.process_iter(["name"]):
+                pname = (proc.info.get("name") or "").lower()
+                if want in pname:
+                    return True, f'process "{name}" running'
+            return False, f'process "{name}" not running'
+        except Exception:
+            # No psutil: fall back to a window-title heuristic rather than
+            # claiming success we cannot substantiate.
+            return self._check_window(name)
+
+    def _check_browser_url(self, fragment: str) -> tuple[bool, str]:
+        """Confirm the default browser is on the expected page.
+
+        Checks the URL Seed Code last opened *and* the live browser window
+        title, so a match on either counts — the title is what actually proves
+        the page loaded, while the URL proves we asked for the right one.
+        """
+        if self._browser is None:
+            from . import browser as browser  # type: ignore
+
+            self._browser = browser
+        want = str(fragment).lower().strip()
+        try:
+            info = str(self._browser.get_page_info() or "")
+        except Exception as exc:
+            return False, f"browser info unavailable: {exc}"
+        if not info:
+            return False, f'no browser page open (expected "{fragment}")'
+        if want in info.lower():
+            return True, f'browser shows "{fragment}"'
+        first_line = info.splitlines()[0] if info.splitlines() else info
+        return False, f'browser does not show "{fragment}" (currently: {first_line})'