use-computer-cli 0.2.0__tar.gz → 0.2.2__tar.gz

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: use-computer-cli
-Version: 0.2.0
+Version: 0.2.2
 Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
 Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
 Author: Bruno Fortunato
@@ -51,7 +51,6 @@ Provides-Extra: local
 Requires-Dist: mss>=10; extra == 'local'
 Requires-Dist: pynput>=1.8; extra == 'local'
 Provides-Extra: tree
-Requires-Dist: pygobject>=3.46; (sys_platform == 'linux') and extra == 'tree'
 Requires-Dist: pyobjc-framework-applicationservices>=10; (sys_platform == 'darwin') and extra == 'tree'
 Requires-Dist: uiautomation>=2.0; (sys_platform == 'win32') and extra == 'tree'
 Provides-Extra: vnc
@@ -84,16 +83,26 @@ with a Python API underneath.
 pip install use-computer-cli            # no backend
 pip install "use-computer-cli[local]"   # drive this machine's display (pynput + mss)
 pip install "use-computer-cli[vnc]"     # drive a remote framebuffer over RFB (vncdotool)
-pip install "use-computer-cli[tree]"    # read the accessibility tree
+pip install "use-computer-cli[tree]"    # read the accessibility tree (Windows, macOS)
 ```
 
 Backends and the accessibility bindings are optional extras, imported lazily, so the package
 installs without them.
 
-The `tree` extra installs one binding per platform: `uiautomation` on Windows,
-`pyobjc-framework-ApplicationServices` on macOS, PyGObject on Linux. **Linux needs a system package
-as well** -- there is no `pyatspi` on PyPI, so the AT-SPI bindings come from `gir1.2-atspi-2.0`
-(plus `python3-pyatspi` on Debian and Ubuntu). The error names both halves when one is missing.
+**On Linux, do not use the `tree` extra.** PyGObject has no Linux wheel, so pip would build it from
+source and fail. The bindings are already on almost every desktop; the virtualenv just has to see
+them:
+
+```bash
+sudo apt install python3-gi gir1.2-atspi-2.0
+python3 -m venv --system-site-packages .venv
+.venv/bin/pip install "use-computer-cli[local]"
+```
+
+`gi` is a compiled extension built for one Python minor version -- Ubuntu 22.04 ships it for 3.10 --
+so the virtualenv has to be the distro's `python3`, not another minor version. Both packages report
+as installed either way, which is why the error message checks and says which case you are in.
+With uv: `uv venv --python /usr/bin/python3 --system-site-packages`.
 
 The distribution is `use-computer-cli` because `use-computer` is taken on PyPI by an unrelated
 project. The command it installs is `use-computer`, and the package it imports is `use_computer`.

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/README.md

@@ -24,16 +24,26 @@ with a Python API underneath.
 pip install use-computer-cli            # no backend
 pip install "use-computer-cli[local]"   # drive this machine's display (pynput + mss)
 pip install "use-computer-cli[vnc]"     # drive a remote framebuffer over RFB (vncdotool)
-pip install "use-computer-cli[tree]"    # read the accessibility tree
+pip install "use-computer-cli[tree]"    # read the accessibility tree (Windows, macOS)
 ```
 
 Backends and the accessibility bindings are optional extras, imported lazily, so the package
 installs without them.
 
-The `tree` extra installs one binding per platform: `uiautomation` on Windows,
-`pyobjc-framework-ApplicationServices` on macOS, PyGObject on Linux. **Linux needs a system package
-as well** -- there is no `pyatspi` on PyPI, so the AT-SPI bindings come from `gir1.2-atspi-2.0`
-(plus `python3-pyatspi` on Debian and Ubuntu). The error names both halves when one is missing.
+**On Linux, do not use the `tree` extra.** PyGObject has no Linux wheel, so pip would build it from
+source and fail. The bindings are already on almost every desktop; the virtualenv just has to see
+them:
+
+```bash
+sudo apt install python3-gi gir1.2-atspi-2.0
+python3 -m venv --system-site-packages .venv
+.venv/bin/pip install "use-computer-cli[local]"
+```
+
+`gi` is a compiled extension built for one Python minor version -- Ubuntu 22.04 ships it for 3.10 --
+so the virtualenv has to be the distro's `python3`, not another minor version. Both packages report
+as installed either way, which is why the error message checks and says which case you are in.
+With uv: `uv venv --python /usr/bin/python3 --system-site-packages`.
 
 The distribution is `use-computer-cli` because `use-computer` is taken on PyPI by an unrelated
 project. The command it installs is `use-computer`, and the package it imports is `use_computer`.

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer-cli"
-version = "0.2.0"
+version = "0.2.2"
 description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -46,10 +46,14 @@ vnc = ["vncdotool>=1.3"]
 #
 # Not dogtail (GPLv2 -- fine to run, wrong to import from a permissively licensed library) and not
 # atomacos (last release 3.3.0, May 2021 -- the same staleness argument that ruled out pyautogui).
+# Linux is deliberately absent. PyGObject has no Linux wheel, so listing it here made
+# `pip install "use-computer-cli[tree]"` build from source, need pycairo and system headers, and
+# fail -- leaving the user with no CLI at all rather than a CLI missing one capability. The AT-SPI
+# bindings come from the distro (python3-gi, gir1.2-atspi-2.0) and are reached with a
+# --system-site-packages virtualenv; the error message says so when they are absent.
 tree = [
     "uiautomation>=2.0; sys_platform == 'win32'",
     "pyobjc-framework-ApplicationServices>=10; sys_platform == 'darwin'",
-    "PyGObject>=3.46; sys_platform == 'linux'",
 ]
 dev = [
     "pytest>=8",

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/src/use_computer/accessibility/atspi.py

@@ -8,15 +8,71 @@ error, because the agent reading it is the one that has to get unstuck.
 
 from __future__ import annotations
 
+import sys
 from contextlib import suppress
+from glob import glob
+from pathlib import Path
 from typing import Any
 
 from use_computer.accessibility import roles
 from use_computer.accessibility.base import require
-from use_computer.errors import UITreeUnavailableError
+from use_computer.errors import UITreeUnavailableError, UseComputerError
 from use_computer.tree import Box, TreeScope, TreeScopeKind, UINode, WindowInfo
 
-SYSTEM_PACKAGE = "gir1.2-atspi-2.0 (and python3-pyatspi on Debian/Ubuntu)"
+#: Where a distro puts PyGObject. The compiled part carries the Python version it was built for,
+#: which is the fact that decides whether any of the advice below will work.
+DISTRO_PATHS = (
+    "/usr/lib/python3/dist-packages/gi",
+    "/usr/lib/python3*/site-packages/gi",
+    "/usr/lib64/python3*/site-packages/gi",
+)
+
+#: What actually gets AT-SPI working on Linux. Not an extra: PyGObject has no Linux wheel, so
+#: asking pip for it builds from source and fails. The distro has it; a virtualenv only has to be
+#: allowed to see it -- and has to be the same Python it was built for.
+INSTALL_HINT = (
+    "install the distro packages and let the virtualenv see them: "
+    "`sudo apt install python3-gi gir1.2-atspi-2.0` then "
+    "`python3 -m venv --system-site-packages .venv`"
+)
+
+
+def distro_python() -> str | None:
+    """The Python version the installed PyGObject was built for, or None if none is installed.
+
+    `gi` is a compiled extension and its shared object names the version:
+    ``_gi.cpython-310-x86_64-linux-gnu.so``. Reading it is what turns "install these packages"
+    -- advice that is useless to someone who already has them -- into the reason it is not working.
+    """
+    for pattern in DISTRO_PATHS:
+        for directory in glob(pattern):
+            for shared_object in glob(f"{directory}/_gi.cpython-*.so"):
+                tag = Path(shared_object).name.split(".")[1]  # cpython-310-x86_64-linux-gnu
+                digits = tag.split("-")[1] if "-" in tag else ""
+                if digits.isdigit() and len(digits) >= 2:
+                    return f"{digits[0]}.{digits[1:]}"
+    return None
+
+
+def system_hint() -> str:
+    """Say what is actually wrong here, not what is usually wrong."""
+    built_for = distro_python()
+    running = f"{sys.version_info.major}.{sys.version_info.minor}"
+    if built_for is None:
+        return INSTALL_HINT
+    if built_for == running:
+        # It is installed for this interpreter, so the virtualenv simply cannot see it.
+        return (
+            f"the distro's PyGObject is installed for Python {built_for}, but this environment "
+            "cannot see it. Recreate it with: "
+            f"`python{built_for} -m venv --system-site-packages .venv`"
+        )
+    return (
+        f"the distro's PyGObject is built for Python {built_for}, but this interpreter is "
+        f"{running}, so --system-site-packages would expose a module it cannot import. Create the "
+        f"environment with the matching interpreter: "
+        f"`python{built_for} -m venv --system-site-packages .venv`"
+    )
 
 #: Milliseconds any single AT-SPI call may take. AT-SPI is D-Bus, and every property read is a
 #: round trip into another process: one application that is wedged, or merely slow to answer,
@@ -36,14 +92,29 @@ class AtspiProvider:
     name = "atspi"
 
     def __init__(self) -> None:
-        gi = require("gi", extra="tree", system=SYSTEM_PACKAGE)
+        gi = require("gi", extra=None, system=system_hint())
         try:
             gi.require_version("Atspi", "2.0")
             from gi.repository import Atspi  # noqa: PLC0415 - deliberately not at module import
         except (ImportError, ValueError) as exc:
             raise UITreeUnavailableError(
-                "the Atspi typelib is not installed.", extra="tree", system=SYSTEM_PACKAGE
+                "the Atspi typelib is not installed.", system=system_hint()
             ) from exc
+        # GLib talks over our stderr from inside the C library, about a stale socket it then
+        # recovers from. An agent reading stderr to understand a failure would see it immediately
+        # before output that is entirely fine. A log handler, not a redirect of fd 2: a redirect
+        # would swallow our own errors during the same window.
+        with suppress(Exception):
+            from gi.repository import GLib  # noqa: PLC0415
+
+            GLib.log_set_handler(
+                "dbind",
+                GLib.LogLevelFlags.LEVEL_MASK
+                | GLib.LogLevelFlags.FLAG_FATAL
+                | GLib.LogLevelFlags.FLAG_RECURSION,
+                lambda *_: None,
+                None,
+            )
         self._atspi = Atspi
         # Older bindings do not expose it; the default timeout then applies.
         with suppress(Exception):
@@ -55,21 +126,30 @@ class AtspiProvider:
     # --- reading -----------------------------------------------------------------------------
 
     def windows(self) -> list[WindowInfo]:
+        return [info for info, _ in self._window_pairs()]
+
+    def _window_pairs(self) -> list[tuple[WindowInfo, Any]]:
+        """Every window, with the accessible it describes, so a match can be acted on."""
         desktop = self._desktop()
-        found: list[WindowInfo] = []
+        found: list[tuple[WindowInfo, Any]] = []
         for app_index, app in enumerate(self._children(desktop)):
             try:
                 pid = self._pid(app)
+                app_name = app.get_name() or None
                 for index, window in enumerate(self._children(app)):
                     states = set(self._states(window))
                     found.append(
-                        WindowInfo(
-                            id=f"0/{app_index}/{index}",
-                            title=window.get_name() or None,
-                            role=roles.atspi_role(window.get_role_name() or ""),
-                            pid=pid if pid > 0 else None,
-                            box=self._box(window),
-                            active="active" in states or "focused" in states,
+                        (
+                            WindowInfo(
+                                id=f"0/{app_index}/{index}",
+                                title=window.get_name() or None,
+                                role=roles.atspi_role(window.get_role_name() or ""),
+                                app=app_name,
+                                pid=pid if pid > 0 else None,
+                                box=self._box(window),
+                                active="active" in states or "focused" in states,
+                            ),
+                            window,
                         )
                     )
             except Exception:
@@ -82,7 +162,9 @@ class AtspiProvider:
         try:
             root = self._root_for(scope)
             return self._build(root, "0", depth)
-        except UITreeUnavailableError:
+        except UseComputerError:
+            # Our own errors already say what to do. Relabelling one as "AT-SPI did not answer"
+            # would hide an ambiguous window behind a transport failure it has nothing to do with.
             raise
         except Exception as exc:
             # A GLib/dbind failure is not a Python error the caller can act on. Turning it into
@@ -109,6 +191,9 @@ class AtspiProvider:
         if scope.kind is TreeScopeKind.ALL:
             return desktop
 
+        if scope.kind is TreeScopeKind.ID and scope.value:
+            return self._by_path(desktop, scope.value)
+
         for app in self._children(desktop):
             # One unresponsive application must not cost the whole snapshot. Skipping it loses
             # that application; letting it raise loses the desktop.
@@ -118,12 +203,6 @@ class AtspiProvider:
                 for window in self._children(app):
                     if scope.kind is TreeScopeKind.FOCUSED and self._is_active(window):
                         return window
-                    if (
-                        scope.kind is TreeScopeKind.TITLE
-                        and scope.value
-                        and scope.value.casefold() in (window.get_name() or "").casefold()
-                    ):
-                        return window
                     if scope.kind is TreeScopeKind.PID:
                         return window
             except Exception:
@@ -138,6 +217,17 @@ class AtspiProvider:
             )
         raise UITreeUnavailableError(f"no window matches {scope.value!r}.")
 
+    def _by_path(self, desktop: Any, path: str) -> Any:
+        """The window an id names. Ids are index paths, so this is a walk."""
+        node = desktop
+        for step in path.split("/")[1:]:
+            children = self._children(node)
+            index = int(step) if step.isdigit() else -1
+            if not 0 <= index < len(children):
+                raise UITreeUnavailableError(f"no window at {path!r}.")
+            node = children[index]
+        return node
+
     def _children(self, obj: Any) -> list[Any]:
         try:
             total = obj.get_child_count()

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/src/use_computer/accessibility/ax.py

@@ -51,6 +51,7 @@ class AxProvider:
         if app is None:
             return []
         focused = self._attr(app, "AXFocusedWindow")
+        app_name = self._attr(app, "AXTitle")
         found: list[WindowInfo] = []
         for index, window in enumerate(self._attr(app, "AXWindows") or []):
             title = self._attr(window, "AXTitle")
@@ -59,6 +60,7 @@ class AxProvider:
                     id=f"0/{index}",
                     title=str(title) if title else None,
                     role=roles.ax_role(str(self._attr(window, "AXRole") or "")),
+                    app=str(app_name) if app_name else None,
                     pid=None,
                     box=self._box(window),
                     active=bool(focused is not None and window == focused),
@@ -88,10 +90,10 @@ class AxProvider:
         app = self._attr(system, "AXFocusedApplication")
         if scope.kind is TreeScopeKind.ALL:
             return app or system
-        if scope.kind is TreeScopeKind.TITLE and scope.value and app is not None:
-            for window in self._attr(app, "AXWindows") or []:
-                title = self._attr(window, "AXTitle") or ""
-                if scope.value.casefold() in str(title).casefold():
+        # A title is resolved to an id above the Protocol, so a provider only ever sees an id.
+        if scope.kind is TreeScopeKind.ID and scope.value and app is not None:
+            for index, window in enumerate(self._attr(app, "AXWindows") or []):
+                if scope.value == f"0/{index}":
                     return window
         focused = self._attr(app, "AXFocusedWindow") if app is not None else None
         return focused or app or system

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/src/use_computer/accessibility/base.py

@@ -54,14 +54,15 @@ class AccessibilityProvider(Protocol):
         """Release whatever the provider holds."""
 
 
-def require(module: str, *, extra: str, system: str | None = None) -> ModuleType:
+def require(module: str, *, extra: str | None, system: str | None = None) -> ModuleType:
     """Import a platform binding, or say exactly what to install.
 
     Called inside a provider's constructor, never at module import: otherwise
     ``use-computer --help`` stops working on a machine without the extra.
 
     Raises:
-        UITreeUnavailableError: naming the extra and, where one is needed, the system package.
+        UITreeUnavailableError: naming the extra where there is one, and the system packages where
+            those are what actually works -- on Linux there is no useful extra to name.
     """
     try:
         return import_module(module)

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/src/use_computer/accessibility/uia.py

@@ -10,6 +10,7 @@ from typing import Any
 
 from use_computer.accessibility import roles
 from use_computer.accessibility.base import require
+from use_computer.errors import UITreeUnavailableError
 from use_computer.tree import Box, TreeScope, TreeScopeKind, UINode, WindowInfo
 
 
@@ -33,6 +34,10 @@ class UiaProvider:
                         id=f"0/{index}",
                         title=window.Name or None,
                         role=roles.uia_role(window.ControlTypeName or ""),
+                        # UI Automation reports the process id and not its name, and turning one
+                        # into the other needs an API this package does not carry. Left empty
+                        # rather than guessed; the pid still tells two windows apart.
+                        app=None,
                         pid=int(window.ProcessId),
                         box=self._box(window),
                         active=bool(window.HasKeyboardFocus),
@@ -48,21 +53,21 @@ class UiaProvider:
 
     def _root_for(self, scope: TreeScope) -> Any:
         auto = self._auto
+        root = auto.GetRootControl()
         if scope.kind is TreeScopeKind.ALL:
-            return auto.GetRootControl()
+            return root
         if scope.kind is TreeScopeKind.FOCUSED:
             control = auto.GetFocusedControl()
-            return self._top_window(control) if control else auto.GetRootControl()
-        for window in auto.GetRootControl().GetChildren():
-            if scope.kind is TreeScopeKind.PID and str(window.ProcessId) == scope.value:
+            return self._top_window(control) if control else root
+        # A title has already been resolved to an id above the Protocol, so only an id or a pid
+        # reaches a provider. Keeping the matching in one place is what stops three platforms
+        # drifting apart on it.
+        for index, window in enumerate(root.GetChildren()):
+            if scope.kind is TreeScopeKind.ID and scope.value == f"0/{index}":
                 return window
-            if (
-                scope.kind is TreeScopeKind.TITLE
-                and scope.value
-                and scope.value.casefold() in (window.Name or "").casefold()
-            ):
+            if scope.kind is TreeScopeKind.PID and str(window.ProcessId) == scope.value:
                 return window
-        return auto.GetRootControl()
+        raise UITreeUnavailableError(f"no window matches {scope.value!r}.")
 
     def _top_window(self, control: Any) -> Any:
         """Walk up to the window the focused control lives in."""

{use_computer_cli-0.2.0 → use_computer_cli-0.2.2}/src/use_computer/actions.py

@@ -186,9 +186,29 @@ class KeyAction(BaseAction):
 
 class ScreenshotAction(BaseAction):
     action: Literal["screenshot"] = "screenshot"
+
+    @field_validator("window", mode="before")
+    @classmethod
+    def _parse_scope(cls, value: Any) -> Any:
+        return TreeScope.parse(value) if isinstance(value, str) else value
+
     out: Path | None = Field(
         default=None, description="Where to write it. None means the screenshot directory."
     )
+    of: str | None = Field(
+        default=None,
+        description="Crop to this node's box. The tree knows where; only what is missing.",
+    )
+    window: TreeScope = Field(
+        default_factory=TreeScope,
+        description="Which tree `of` is an id in. An id means nothing without its scope.",
+    )
+    pad: int = Field(
+        default=0,
+        ge=0,
+        description="Grow the crop by this many pixels each side; a control's box often "
+        "excludes the label beside it.",
+    )
 
 
 class TreeAction(BaseAction):