use-computer-cli 0.2.2__tar.gz → 0.3.1__tar.gz

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: use-computer-cli
-Version: 0.2.2
+Version: 0.3.1
 Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
 Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
 Author: Bruno Fortunato
@@ -52,6 +52,7 @@ Requires-Dist: mss>=10; extra == 'local'
 Requires-Dist: pynput>=1.8; extra == 'local'
 Provides-Extra: tree
 Requires-Dist: pyobjc-framework-applicationservices>=10; (sys_platform == 'darwin') and extra == 'tree'
+Requires-Dist: python-xlib>=0.33; (sys_platform == 'linux') and extra == 'tree'
 Requires-Dist: uiautomation>=2.0; (sys_platform == 'win32') and extra == 'tree'
 Provides-Extra: vnc
 Requires-Dist: vncdotool>=1.3; extra == 'vnc'

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer-cli"
-version = "0.2.2"
+version = "0.3.1"
 description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -52,6 +52,11 @@ vnc = ["vncdotool>=1.3"]
 # bindings come from the distro (python3-gi, gir1.2-atspi-2.0) and are reached with a
 # --system-site-packages virtualenv; the error message says so when they are absent.
 tree = [
+    # python-xlib reads _NET_ACTIVE_WINDOW, the only reliable answer to "which window is in
+    # front" -- AT-SPI reports `active` per application and marks several at once. A pure-Python
+    # wheel with no build step, which is exactly what PyGObject was not, so it does not
+    # reintroduce BUG-003. Imported softly: absent, or on Wayland, the tool falls back.
+    "python-xlib>=0.33; sys_platform == 'linux'",
     "uiautomation>=2.0; sys_platform == 'win32'",
     "pyobjc-framework-ApplicationServices>=10; sys_platform == 'darwin'",
 ]
@@ -102,6 +107,7 @@ module = [
     "ApplicationServices.*",
     "Quartz.*",
     "gi.*",
+    "Xlib.*",
     "uiautomation.*",
     "tomli",
 ]

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/src/use_computer/accessibility/atspi.py

@@ -17,7 +17,14 @@ from typing import Any
 from use_computer.accessibility import roles
 from use_computer.accessibility.base import require
 from use_computer.errors import UITreeUnavailableError, UseComputerError
-from use_computer.tree import Box, TreeScope, TreeScopeKind, UINode, WindowInfo
+from use_computer.tree import (
+    ActiveWindow,
+    Box,
+    TreeScope,
+    TreeScopeKind,
+    UINode,
+    WindowInfo,
+)
 
 #: Where a distro puts PyGObject. The compiled part carries the Python version it was built for,
 #: which is the fact that decides whether any of the advice below will work.
@@ -126,7 +133,31 @@ class AtspiProvider:
     # --- reading -----------------------------------------------------------------------------
 
     def windows(self) -> list[WindowInfo]:
-        return [info for info, _ in self._window_pairs()]
+        found = [info for info, _ in self._window_pairs()]
+        if any(info.active for info in found):
+            return found
+        # Nothing claims `active`. Some desktops only ever report `focused`, so fall back to it
+        # rather than returning a list with no mark at all -- but never mix the two: `focused` on
+        # this desktop marks the shell, which is not a window anybody wants to act in.
+        return [
+            info.model_copy(update={"active": info.id in self._focused_ids()})
+            for info in found
+        ]
+
+    def _focused_ids(self) -> frozenset[str]:
+        ids: set[str] = set()
+        try:
+            desktop = self._desktop()
+            for app_index, app in enumerate(self._children(desktop)):
+                try:
+                    for index, window in enumerate(self._children(app)):
+                        if "focused" in set(self._states(window)):
+                            ids.add(f"0/{app_index}/{index}")
+                except Exception:
+                    continue
+        except Exception:
+            return frozenset()
+        return frozenset(ids)
 
     def _window_pairs(self) -> list[tuple[WindowInfo, Any]]:
         """Every window, with the accessible it describes, so a match can be acted on."""
@@ -147,7 +178,7 @@ class AtspiProvider:
                                 app=app_name,
                                 pid=pid if pid > 0 else None,
                                 box=self._box(window),
-                                active="active" in states or "focused" in states,
+                                active="active" in states,
                             ),
                             window,
                         )
@@ -262,7 +293,7 @@ class AtspiProvider:
 
     def _is_active(self, window: Any) -> bool:
         states = set(self._states(window))
-        return "active" in states or "focused" in states
+        return "active" in states
 
     #: AT-SPI reports an element that is not currently rendered at INT_MIN with a 1x1 size --
     #: the items of a closed menu, for instance. That is a sentinel, not a position, and letting
@@ -378,6 +409,56 @@ class AtspiProvider:
                 return bool(interface.do_action(index))
         return False
 
+    def active_window(self) -> ActiveWindow | None:
+        """Ask the window manager, because AT-SPI cannot answer this.
+
+        `_NET_ACTIVE_WINDOW` on the root window names exactly one window; AT-SPI's `active` state
+        is per application and marked three at once on the desktop that produced the bug. Imported
+        softly on purpose: no python-xlib, or a Wayland session that publishes no such property,
+        and the caller falls back to the flags.
+        """
+        try:
+            from Xlib import X, display
+        except ImportError:
+            return None
+        try:
+            connection = display.Display()
+        except Exception:
+            return None  # no X server: Wayland, or no session at all
+        try:
+            root = connection.screen().root
+            active = root.get_full_property(
+                connection.intern_atom("_NET_ACTIVE_WINDOW"), X.AnyPropertyType
+            )
+            if active is None or not active.value:
+                return None
+            window = connection.create_resource_object("window", active.value[0])
+            pid = window.get_full_property(
+                connection.intern_atom("_NET_WM_PID"), X.AnyPropertyType
+            )
+            name = window.get_full_property(
+                connection.intern_atom("_NET_WM_NAME"), connection.intern_atom("UTF8_STRING")
+            )
+            return ActiveWindow(
+                pid=int(pid.value[0]) if pid is not None and pid.value else None,
+                title=name.value.decode("utf-8", "replace") if name is not None else None,
+            )
+        except Exception:
+            # A hint that fails is not an error: the caller has a working fallback.
+            return None
+        finally:
+            with suppress(Exception):
+                connection.close()
+
+    def activate(self, window_id: str) -> bool:
+        """AT-SPI offers no raise for a window.
+
+        A window accessible reports no actions at all -- verified: `It supports: none` -- so there
+        is nothing native to call here. False tells the caller to focus a descendant instead,
+        which is what actually moves `_NET_ACTIVE_WINDOW`.
+        """
+        return False
+
     def close(self) -> None:
         self._index = {}
 

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/src/use_computer/accessibility/ax.py

@@ -15,7 +15,14 @@ from typing import Any
 from use_computer.accessibility import roles
 from use_computer.accessibility.base import require
 from use_computer.errors import PermissionDeniedError
-from use_computer.tree import Box, TreeScope, TreeScopeKind, UINode, WindowInfo
+from use_computer.tree import (
+    ActiveWindow,
+    Box,
+    TreeScope,
+    TreeScopeKind,
+    UINode,
+    WindowInfo,
+)
 
 #: kAXErrorAPIDisabled -- the process is not trusted for accessibility.
 API_DISABLED = -25211
@@ -197,6 +204,21 @@ class AxProvider:
         roles.COLLAPSE: "AXPress",
     }
 
+    def active_window(self) -> ActiveWindow | None:
+        """This platform's own `active` flag is already per window, so there is nothing to add."""
+        return None
+
+    def activate(self, window_id: str) -> bool:
+        """macOS windows do accept a raise of their own: AXRaise."""
+        element = self._index.get(window_id)
+        if element is None:
+            return False
+        try:
+            status = self._api.AXUIElementPerformAction(element, "AXRaise")
+        except Exception:
+            return False
+        return bool(status == 0)
+
     def perform(self, node_id: str, action: str, value: str | None) -> bool:
         element = self._index.get(node_id)
         if element is None:

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/src/use_computer/accessibility/base.py

@@ -15,7 +15,7 @@ from types import ModuleType
 from typing import Protocol, runtime_checkable
 
 from use_computer.errors import UITreeUnavailableError
-from use_computer.tree import TreeScope, UINode, WindowInfo
+from use_computer.tree import ActiveWindow, TreeScope, UINode, WindowInfo
 
 
 @runtime_checkable
@@ -50,6 +50,24 @@ class AccessibilityProvider(Protocol):
         nothing at all. ``False`` is what makes ``--via auto`` fall back to a coordinate click.
         """
 
+    def activate(self, window_id: str) -> bool:
+        """Bring a window forward, natively, if this platform has a way.
+
+        ``True`` means the window was raised. ``False`` is not a failure: it means this platform
+        offers no raise for a window object, and the caller should focus a descendant instead --
+        which raises the top-level window on all three. AT-SPI is the ``False`` case; a window
+        accessible there reports no actions at all.
+        """
+
+    def active_window(self) -> ActiveWindow | None:
+        """What the window manager says is in front, as a pid and a title.
+
+        A *hint*. ``None`` means this platform has nothing better than the per-window flags, which
+        is the honest answer on Wayland or without the optional binding. Matching it to a window is
+        policy and lives in ``selectors.mark_active``: a rule that lives in a provider is a rule
+        the other two platforms drift from, which has already happened once here.
+        """
+
     def close(self) -> None:
         """Release whatever the provider holds."""
 

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/src/use_computer/accessibility/uia.py

@@ -11,7 +11,14 @@ from typing import Any
 from use_computer.accessibility import roles
 from use_computer.accessibility.base import require
 from use_computer.errors import UITreeUnavailableError
-from use_computer.tree import Box, TreeScope, TreeScopeKind, UINode, WindowInfo
+from use_computer.tree import (
+    ActiveWindow,
+    Box,
+    TreeScope,
+    TreeScopeKind,
+    UINode,
+    WindowInfo,
+)
 
 
 class UiaProvider:
@@ -167,6 +174,21 @@ class UiaProvider:
 
     # --- acting ------------------------------------------------------------------------------
 
+    def active_window(self) -> ActiveWindow | None:
+        """This platform's own `active` flag is already per window, so there is nothing to add."""
+        return None
+
+    def activate(self, window_id: str) -> bool:
+        """Windows has a native raise: `SetActive` on the top-level control."""
+        control = self._index.get(window_id)
+        if control is None:
+            return False
+        try:
+            return bool(control.SetActive())
+        except Exception:
+            # Not every control exposes it, and the caller has a working fallback.
+            return False
+
     def perform(self, node_id: str, action: str, value: str | None) -> bool:
         control = self._index.get(node_id)
         if control is None:

{use_computer_cli-0.2.2 → use_computer_cli-0.3.1}/src/use_computer/actions.py

@@ -7,6 +7,7 @@ in actuation units and canonical key names.
 
 from __future__ import annotations
 
+from difflib import get_close_matches
 from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, Literal
@@ -48,7 +49,25 @@ class BaseAction(BaseModel):
         return None
 
 
-class _Positioned(BaseAction):
+class _InAWindow(BaseAction):
+    """An action whose coordinate belongs to a particular window.
+
+    A coordinate lands on whatever window is in front. That is what a bare coordinate has always
+    meant, and it is fine until the agent meant a particular window -- which is most of the time.
+    Naming it here is how the caller says so, and the window is brought forward before the
+    coordinate is sent. Measured: two drags aimed at a canvas selected text in a terminal instead,
+    and `--verify` then confirmed the wrong action with the wrong evidence.
+    """
+
+    window: TreeScope | None = None
+
+    @field_validator("window", mode="before")
+    @classmethod
+    def _parse_scope(cls, value: Any) -> Any:
+        return TreeScope.parse(value) if isinstance(value, str) else value
+
+
+class _Positioned(_InAWindow):
     """An action that may carry a coordinate. Omitting it acts where the pointer already is."""
 
     x: int | None = None
@@ -130,7 +149,7 @@ class RightClickAction(_PositionedOrSelected):
     action: Literal["right_click"] = "right_click"
 
 
-class DragAction(BaseAction):
+class DragAction(_InAWindow):
     action: Literal["drag"] = "drag"
     from_x: int
     from_y: int
@@ -297,6 +316,23 @@ class ShowMenuAction(_ElementAction):
     action: Literal["show_menu"] = "show_menu"
 
 
+class ActivateAction(BaseAction):
+    """Bring a window forward and give it keyboard focus.
+
+    The only action whose target is a window rather than an element or a point. A window object
+    exposes no actions of its own on any of the three platforms -- AT-SPI answers `It supports:
+    none` -- so it is done by focusing a descendant, which raises the top-level window everywhere.
+    """
+
+    action: Literal["activate"] = "activate"
+    window: TreeScope
+
+    @field_validator("window", mode="before")
+    @classmethod
+    def _parse_scope(cls, value: Any) -> Any:
+        return TreeScope.parse(value) if isinstance(value, str) else value
+
+
 Action = Annotated[
     MoveAction
     | ClickAction
@@ -315,7 +351,8 @@ Action = Annotated[
     | CollapseAction
     | SelectAction
     | SetValueAction
-    | ShowMenuAction,
+    | ShowMenuAction
+    | ActivateAction,
     Field(discriminator="action"),
 ]
 
@@ -339,6 +376,72 @@ def selector_of(action: Action) -> NodeSelector | None:
 #: Parses a batch file: a JSON array of action objects, discriminated on `action`.
 ActionListAdapter: TypeAdapter[list[Action]] = TypeAdapter(list[Action])
 
+#: Every action name, in the spelling the discriminator uses.
+ACTION_NAMES: tuple[str, ...] = (
+    "move",
+    "click",
+    "double_click",
+    "right_click",
+    "drag",
+    "scroll",
+    "type",
+    "key",
+    "screenshot",
+    "tree",
+    "windows",
+    "focus",
+    "toggle",
+    "expand",
+    "collapse",
+    "select",
+    "set_value",
+    "show_menu",
+    "activate",
+)
+
+
+def _cli_spelling(name: str) -> str:
+    """How this action is spelled as a command: `set_value` is typed `set-value`."""
+    return name.replace("_", "-")
+
+
+def normalise_action_names(data: Any) -> Any:
+    """Accept the CLI's own spelling of an action name inside a batch.
+
+    The command is `use-computer set-value`; the batch wanted `set_value`. Same action, two
+    spellings, and nothing said so -- an agent that has just read `set-value --help` has no reason
+    to expect a different name three lines later.
+    """
+    if not isinstance(data, list):
+        return data
+    out = []
+    for item in data:
+        if isinstance(item, dict) and isinstance(item.get("action"), str):
+            item = {**item, "action": item["action"].replace("-", "_")}
+        out.append(item)
+    return out
+
+
+def check_action_names(data: Any) -> None:
+    """Fail on an unknown action with a sentence, before pydantic offers its union.
+
+    The union's own error is four hundred characters naming every variant except the one the
+    caller should have written. This is a message to somebody mid-task who cannot see the code.
+    """
+    if not isinstance(data, list):
+        return
+    for index, item in enumerate(data):
+        if not isinstance(item, dict):
+            continue
+        name = item.get("action")
+        if not isinstance(name, str) or name in ACTION_NAMES:
+            continue
+        close = get_close_matches(name, ACTION_NAMES, n=1, cutoff=0.6)
+        suggestion = f" Did you mean {_cli_spelling(close[0])!r}?" if close else ""
+        raise ValueError(
+            f"unknown action {_cli_spelling(name)!r} at index {index}.{suggestion}"
+        )
+
 #: Parses a single action object.
 ActionAdapter: TypeAdapter[Action] = TypeAdapter(Action)