use-computer-cli 0.2.2__tar.gz → 0.3.0__tar.gz

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: use-computer-cli
-Version: 0.2.2
+Version: 0.3.0
 Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
 Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
 Author: Bruno Fortunato

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer-cli"
-version = "0.2.2"
+version = "0.3.0"
 description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
 readme = "README.md"
 requires-python = ">=3.10"

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/src/use_computer/accessibility/atspi.py

@@ -126,7 +126,31 @@ class AtspiProvider:
     # --- reading -----------------------------------------------------------------------------
 
     def windows(self) -> list[WindowInfo]:
-        return [info for info, _ in self._window_pairs()]
+        found = [info for info, _ in self._window_pairs()]
+        if any(info.active for info in found):
+            return found
+        # Nothing claims `active`. Some desktops only ever report `focused`, so fall back to it
+        # rather than returning a list with no mark at all -- but never mix the two: `focused` on
+        # this desktop marks the shell, which is not a window anybody wants to act in.
+        return [
+            info.model_copy(update={"active": info.id in self._focused_ids()})
+            for info in found
+        ]
+
+    def _focused_ids(self) -> frozenset[str]:
+        ids: set[str] = set()
+        try:
+            desktop = self._desktop()
+            for app_index, app in enumerate(self._children(desktop)):
+                try:
+                    for index, window in enumerate(self._children(app)):
+                        if "focused" in set(self._states(window)):
+                            ids.add(f"0/{app_index}/{index}")
+                except Exception:
+                    continue
+        except Exception:
+            return frozenset()
+        return frozenset(ids)
 
     def _window_pairs(self) -> list[tuple[WindowInfo, Any]]:
         """Every window, with the accessible it describes, so a match can be acted on."""
@@ -147,7 +171,7 @@ class AtspiProvider:
                                 app=app_name,
                                 pid=pid if pid > 0 else None,
                                 box=self._box(window),
-                                active="active" in states or "focused" in states,
+                                active="active" in states,
                             ),
                             window,
                         )
@@ -262,7 +286,7 @@ class AtspiProvider:
 
     def _is_active(self, window: Any) -> bool:
         states = set(self._states(window))
-        return "active" in states or "focused" in states
+        return "active" in states
 
     #: AT-SPI reports an element that is not currently rendered at INT_MIN with a 1x1 size --
     #: the items of a closed menu, for instance. That is a sentinel, not a position, and letting
@@ -378,6 +402,15 @@ class AtspiProvider:
                 return bool(interface.do_action(index))
         return False
 
+    def activate(self, window_id: str) -> bool:
+        """AT-SPI offers no raise for a window.
+
+        A window accessible reports no actions at all -- verified: `It supports: none` -- so there
+        is nothing native to call here. False tells the caller to focus a descendant instead,
+        which is what actually moves `_NET_ACTIVE_WINDOW`.
+        """
+        return False
+
     def close(self) -> None:
         self._index = {}
 

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/src/use_computer/accessibility/ax.py

@@ -197,6 +197,17 @@ class AxProvider:
         roles.COLLAPSE: "AXPress",
     }
 
+    def activate(self, window_id: str) -> bool:
+        """macOS windows do accept a raise of their own: AXRaise."""
+        element = self._index.get(window_id)
+        if element is None:
+            return False
+        try:
+            status = self._api.AXUIElementPerformAction(element, "AXRaise")
+        except Exception:
+            return False
+        return bool(status == 0)
+
     def perform(self, node_id: str, action: str, value: str | None) -> bool:
         element = self._index.get(node_id)
         if element is None:

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/src/use_computer/accessibility/base.py

@@ -50,6 +50,15 @@ class AccessibilityProvider(Protocol):
         nothing at all. ``False`` is what makes ``--via auto`` fall back to a coordinate click.
         """
 
+    def activate(self, window_id: str) -> bool:
+        """Bring a window forward, natively, if this platform has a way.
+
+        ``True`` means the window was raised. ``False`` is not a failure: it means this platform
+        offers no raise for a window object, and the caller should focus a descendant instead --
+        which raises the top-level window on all three. AT-SPI is the ``False`` case; a window
+        accessible there reports no actions at all.
+        """
+
     def close(self) -> None:
         """Release whatever the provider holds."""
 

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/src/use_computer/accessibility/uia.py

@@ -167,6 +167,17 @@ class UiaProvider:
 
     # --- acting ------------------------------------------------------------------------------
 
+    def activate(self, window_id: str) -> bool:
+        """Windows has a native raise: `SetActive` on the top-level control."""
+        control = self._index.get(window_id)
+        if control is None:
+            return False
+        try:
+            return bool(control.SetActive())
+        except Exception:
+            # Not every control exposes it, and the caller has a working fallback.
+            return False
+
     def perform(self, node_id: str, action: str, value: str | None) -> bool:
         control = self._index.get(node_id)
         if control is None:

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/src/use_computer/actions.py

@@ -7,6 +7,7 @@ in actuation units and canonical key names.
 
 from __future__ import annotations
 
+from difflib import get_close_matches
 from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, Literal
@@ -48,7 +49,25 @@ class BaseAction(BaseModel):
         return None
 
 
-class _Positioned(BaseAction):
+class _InAWindow(BaseAction):
+    """An action whose coordinate belongs to a particular window.
+
+    A coordinate lands on whatever window is in front. That is what a bare coordinate has always
+    meant, and it is fine until the agent meant a particular window -- which is most of the time.
+    Naming it here is how the caller says so, and the window is brought forward before the
+    coordinate is sent. Measured: two drags aimed at a canvas selected text in a terminal instead,
+    and `--verify` then confirmed the wrong action with the wrong evidence.
+    """
+
+    window: TreeScope | None = None
+
+    @field_validator("window", mode="before")
+    @classmethod
+    def _parse_scope(cls, value: Any) -> Any:
+        return TreeScope.parse(value) if isinstance(value, str) else value
+
+
+class _Positioned(_InAWindow):
     """An action that may carry a coordinate. Omitting it acts where the pointer already is."""
 
     x: int | None = None
@@ -130,7 +149,7 @@ class RightClickAction(_PositionedOrSelected):
     action: Literal["right_click"] = "right_click"
 
 
-class DragAction(BaseAction):
+class DragAction(_InAWindow):
     action: Literal["drag"] = "drag"
     from_x: int
     from_y: int
@@ -297,6 +316,23 @@ class ShowMenuAction(_ElementAction):
     action: Literal["show_menu"] = "show_menu"
 
 
+class ActivateAction(BaseAction):
+    """Bring a window forward and give it keyboard focus.
+
+    The only action whose target is a window rather than an element or a point. A window object
+    exposes no actions of its own on any of the three platforms -- AT-SPI answers `It supports:
+    none` -- so it is done by focusing a descendant, which raises the top-level window everywhere.
+    """
+
+    action: Literal["activate"] = "activate"
+    window: TreeScope
+
+    @field_validator("window", mode="before")
+    @classmethod
+    def _parse_scope(cls, value: Any) -> Any:
+        return TreeScope.parse(value) if isinstance(value, str) else value
+
+
 Action = Annotated[
     MoveAction
     | ClickAction
@@ -315,7 +351,8 @@ Action = Annotated[
     | CollapseAction
     | SelectAction
     | SetValueAction
-    | ShowMenuAction,
+    | ShowMenuAction
+    | ActivateAction,
     Field(discriminator="action"),
 ]
 
@@ -339,6 +376,72 @@ def selector_of(action: Action) -> NodeSelector | None:
 #: Parses a batch file: a JSON array of action objects, discriminated on `action`.
 ActionListAdapter: TypeAdapter[list[Action]] = TypeAdapter(list[Action])
 
+#: Every action name, in the spelling the discriminator uses.
+ACTION_NAMES: tuple[str, ...] = (
+    "move",
+    "click",
+    "double_click",
+    "right_click",
+    "drag",
+    "scroll",
+    "type",
+    "key",
+    "screenshot",
+    "tree",
+    "windows",
+    "focus",
+    "toggle",
+    "expand",
+    "collapse",
+    "select",
+    "set_value",
+    "show_menu",
+    "activate",
+)
+
+
+def _cli_spelling(name: str) -> str:
+    """How this action is spelled as a command: `set_value` is typed `set-value`."""
+    return name.replace("_", "-")
+
+
+def normalise_action_names(data: Any) -> Any:
+    """Accept the CLI's own spelling of an action name inside a batch.
+
+    The command is `use-computer set-value`; the batch wanted `set_value`. Same action, two
+    spellings, and nothing said so -- an agent that has just read `set-value --help` has no reason
+    to expect a different name three lines later.
+    """
+    if not isinstance(data, list):
+        return data
+    out = []
+    for item in data:
+        if isinstance(item, dict) and isinstance(item.get("action"), str):
+            item = {**item, "action": item["action"].replace("-", "_")}
+        out.append(item)
+    return out
+
+
+def check_action_names(data: Any) -> None:
+    """Fail on an unknown action with a sentence, before pydantic offers its union.
+
+    The union's own error is four hundred characters naming every variant except the one the
+    caller should have written. This is a message to somebody mid-task who cannot see the code.
+    """
+    if not isinstance(data, list):
+        return
+    for index, item in enumerate(data):
+        if not isinstance(item, dict):
+            continue
+        name = item.get("action")
+        if not isinstance(name, str) or name in ACTION_NAMES:
+            continue
+        close = get_close_matches(name, ACTION_NAMES, n=1, cutoff=0.6)
+        suggestion = f" Did you mean {_cli_spelling(close[0])!r}?" if close else ""
+        raise ValueError(
+            f"unknown action {_cli_spelling(name)!r} at index {index}.{suggestion}"
+        )
+
 #: Parses a single action object.
 ActionAdapter: TypeAdapter[Action] = TypeAdapter(Action)
 

{use_computer_cli-0.2.2 → use_computer_cli-0.3.0}/src/use_computer/cli.py

@@ -27,6 +27,7 @@ from use_computer import render
 from use_computer.actions import (
     Action,
     ActionListAdapter,
+    ActivateAction,
     ClickAction,
     CollapseAction,
     DoubleClickAction,
@@ -47,6 +48,8 @@ from use_computer.actions import (
     TreeAction,
     TypeAction,
     WindowsAction,
+    check_action_names,
+    normalise_action_names,
 )
 from use_computer.config import (
     ResolvedConfig,
@@ -81,6 +84,9 @@ EXIT_USAGE = 2
 #: The command a bare invocation means. `use-computer actions.json` and `use-computer -` work.
 DEFAULT_COMMAND = "batch"
 
+#: How much typed text a result line carries. Enough to recognise it, not enough to become a log.
+TYPED_TEXT_LIMIT = 48
+
 app = typer.Typer(
     add_completion=False,
     no_args_is_help=True,
@@ -98,7 +104,9 @@ def _version_callback(value: bool) -> None:
     if value:
         from use_computer import __version__
 
-        _emit({"version": __version__})
+        # Text, like everything else. A contract is worth what its least consistent command is
+        # worth, and `--version` answering in JSON was the cheapest possible way to break it.
+        _write(f"use-computer {__version__}")
         raise typer.Exit(EXIT_OK)
 
 
@@ -177,14 +185,24 @@ def _required(value: Any, flag: str) -> Any:
 
 
 def _build(
-    kind: Any, selector: NodeSelector | None, via: Via, **fields: Any
+    kind: Any,
+    selector: NodeSelector | None,
+    via: Via,
+    *,
+    window: str | None = None,
+    **fields: Any,
 ) -> Action:
     """Construct an action from either a coordinate or an element -- never both.
 
     Choosing between them would be exactly the kind of silent reinterpretation this tool refuses
     to do with coordinate spaces, so a caller that gives both is told to pick one.
+
+    `--window` is not part of that choice: on a coordinate it names the window the coordinate
+    belongs to, which is brought forward before the coordinate is sent.
     """
     if selector is None:
+        if window is not None:
+            fields["window"] = window
         return kind(**fields)  # type: ignore[no-any-return]
     if fields.get("x") is not None or fields.get("y") is not None:
         _err.print(
@@ -196,6 +214,11 @@ def _build(
     return kind(selector=selector, via=via, **fields)  # type: ignore[no-any-return]
 
 
+def _scope(window: str | None) -> TreeScope | None:
+    """`--window` as a scope. `None` keeps a bare coordinate meaning what it always meant."""
+    return TreeScope.parse(window) if window is not None else None
+
+
 def _selector(
     node_id: str | None,
     role: str | None,
@@ -248,7 +271,12 @@ def _say(template: str, **values: Any) -> None:
 
 def _emit(payload: Any) -> None:
     """stdout is JSON and nothing else."""
-    sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n")
+    _write(json.dumps(payload, ensure_ascii=False))
+
+
+def _write(text: str) -> None:
+    """stdout, unstyled. Never through rich: it soft-wraps and eats square brackets."""
+    sys.stdout.write(text + "\n")
     sys.stdout.flush()
 
 
@@ -311,6 +339,13 @@ def _text_lines(result: Any) -> str:
         if item.tree is not None:
             if item.tree.text:
                 chunks.append(item.tree.text)
+                if item.tree.limited_by:
+                    # Said, not implied: a tree with nothing under its root looks exactly like an
+                    # application that exposes nothing, and the two call for opposite moves.
+                    chunks.append(
+                        f"# nothing below the root: {item.tree.limited_by} emptied it, "
+                        "not the application"
+                    )
             elif item.tree.reason is not None:
                 shot = item.tree.screenshot
                 where = f"; screenshot at {shot.path}" if shot is not None else ""
@@ -327,10 +362,20 @@ def _text_lines(result: Any) -> str:
         if item.action.action == "screenshot" and item.screenshot is not None:
             # The path is the answer, and the only part of it worth any tokens.
             shot = item.screenshot
-            where = f" {shot.box[2]}x{shot.box[3]} of {shot.of}" if shot.box else ""
+            where = (
+                f" {shot.box[2]}x{shot.box[3]} of {shot.of}"
+                if shot.box
+                else f" {shot.width}x{shot.height}"
+            )
             chunks.append(f"{_shorten(shot.path, folder)}{where}")
             continue
-        what = item.action.action
+        if item.action.action == "activate" and item.activated:
+            chunks.append(
+                f"{'would activate ' if not item.performed else 'activated '}{item.activated}"
+                f" \u2014 {item.duration_ms:.0f} ms"
+            )
+            continue
+        what = item.action.action + _payload(item)
         if item.matched is not None:
             what += f" {item.matched.role}"
             if item.matched.name:
@@ -339,7 +384,9 @@ def _text_lines(result: Any) -> str:
         how = ""
         if item.via is not None:
             how = " via the platform API" if item.via.value == "action" else " via a coordinate"
-        elif item.resolved is not None:
+            if item.via.value != "action" and item.resolved is not None:
+                how += f" ({item.resolved.x}, {item.resolved.y})"
+        elif item.resolved is not None and item.resolved_from is None:
             how = f" at ({item.resolved.x}, {item.resolved.y})"
         done = "would " if not item.performed else ""
         # --verify exists to give feedback, so its answer belongs on the line. `unchanged` is the
@@ -347,11 +394,7 @@ def _text_lines(result: Any) -> str:
         # clicking the same wrong pixel forever.
         verified = ""
         if item.change is not None:
-            verified = (
-                f" \u2014 changed {item.change.magnitude:.0%}"
-                if item.change.changed
-                else " \u2014 unchanged"
-            )
+            verified = f" \u2014 {_changed(item.change)}"
         chunks.append(f"{done}{what}{how}{verified} \u2014 {item.duration_ms:.0f} ms")
         if item.screenshot is not None and item.screenshot.path is not None:
             # Already captured and already paid for. Saying where saves the agent asking again,
@@ -362,6 +405,44 @@ def _text_lines(result: Any) -> str:
     return "\n".join(chunks)
 
 
+def _payload(item: Any) -> str:
+    """What this action actually did, on the line that is its only record.
+
+    `key` without its combination, `type` without its text, a `drag` with one of its two points:
+    each is blank in exactly the place an agent looks when the screen does not match its model.
+    Eleven such lines could not be reconstructed into what a batch had done.
+    """
+    action = item.action
+    name = action.action
+    if name == "key":
+        return f" {action.combo}"
+    if name == "type":
+        text = action.text
+        shown = text if len(text) <= TYPED_TEXT_LIMIT else text[: TYPED_TEXT_LIMIT - 1] + "\u2026"
+        # The count is what catches a truncated or a doubled paste; the text is what identifies it.
+        return f" {len(text)} chars {shown!r}"
+    if name == "drag" and item.resolved is not None and item.resolved_from is not None:
+        start, end = item.resolved_from, item.resolved
+        return f" ({start.x}, {start.y}) \u2192 ({end.x}, {end.y})"
+    if name == "move" and item.resolved is not None:
+        return f" ({item.resolved.x}, {item.resolved.y})"
+    return ""
+
+
+def _changed(change: Any) -> str:
+    """Where the screen changed, which is what an agent can act on.
+
+    A percentage cannot be compared against what was expected to happen, and rounds a real change
+    of a few thousand pixels to `changed 0%`, which reads as a denial.
+    """
+    if not change.changed:
+        return "unchanged"
+    if change.bbox is None:
+        return "changed"
+    left, top, right, bottom = change.bbox
+    return f"changed {right - left}x{bottom - top} at {left},{top}"
+
+
 def _shorten(path: Path | None, folder: Path | None) -> str:
     """The filename when the directory has already been named, the whole path otherwise."""
     if path is None:
@@ -457,6 +538,7 @@ def _run(
 def move(
     x: Annotated[int, typer.Option("--x", help="X coordinate.")],
     y: Annotated[int, typer.Option("--y", help="Y coordinate.")],
+    window: WindowOption = None,
     use: UseOption = None,
     space: SpaceOption = None,
     delay: DelayOption = None,
@@ -467,7 +549,9 @@ def move(
 ) -> None:
     """Move the pointer."""
     config = _config(use, space=space, delay=delay, dry_run=dry_run, verify=verify, verbose=verbose)
-    _run([MoveAction(x=x, y=y, space=space)], config, verbose, fmt=format)
+    _run(
+        [MoveAction(x=x, y=y, space=space, window=_scope(window))], config, verbose, fmt=format
+    )
 
 
 @app.command()
@@ -494,7 +578,7 @@ def click(
     config = _config(use, space=space, delay=delay, dry_run=dry_run, verify=verify, verbose=verbose)
     selector = _selector(id, role, name, exact, nth, window)
     _run(
-        [_build(ClickAction, selector, via, x=x, y=y, space=space, button=button)],
+        [_build(ClickAction, selector, via, window=window, x=x, y=y, space=space, button=button)],
         config,
         verbose,
         fmt=format,
@@ -523,7 +607,7 @@ def double_click(
     """Double-click an element or a coordinate."""
     config = _config(use, space=space, delay=delay, dry_run=dry_run, verify=verify, verbose=verbose)
     selector = _selector(id, role, name, exact, nth, window)
-    action = _build(DoubleClickAction, selector, via, x=x, y=y, space=space)
+    action = _build(DoubleClickAction, selector, via, window=window, x=x, y=y, space=space)
     _run([action], config, verbose, fmt=format)
 
 
@@ -549,7 +633,7 @@ def right_click(
     """Click an element or a coordinate with the secondary button."""
     config = _config(use, space=space, delay=delay, dry_run=dry_run, verify=verify, verbose=verbose)
     selector = _selector(id, role, name, exact, nth, window)
-    action = _build(RightClickAction, selector, via, x=x, y=y, space=space)
+    action = _build(RightClickAction, selector, via, window=window, x=x, y=y, space=space)
     _run([action], config, verbose, fmt=format)
 
 
@@ -560,6 +644,7 @@ def drag(
     to_x: Annotated[int | None, typer.Option("--to-x")] = None,
     to_y: Annotated[int | None, typer.Option("--to-y")] = None,
     button: Annotated[MouseButton, typer.Option("--button")] = MouseButton.LEFT,
+    window: WindowOption = None,
     use: UseOption = None,
     space: SpaceOption = None,
     delay: DelayOption = None,
@@ -575,7 +660,8 @@ def drag(
     to_y = _required(to_y, "--to-y")
     config = _config(use, space=space, delay=delay, dry_run=dry_run, verify=verify, verbose=verbose)
     action = DragAction(
-        from_x=from_x, from_y=from_y, to_x=to_x, to_y=to_y, space=space, button=button
+        from_x=from_x, from_y=from_y, to_x=to_x, to_y=to_y, space=space, button=button,
+        window=_scope(window),
     )
     _run([action], config, verbose, fmt=format)
 
@@ -608,8 +694,8 @@ def scroll(
     _run(
         [
             _build(
-                ScrollAction, selector, via, amount=amount, direction=direction, x=x, y=y,
-                space=space,
+                ScrollAction, selector, via, window=window, amount=amount,
+                direction=direction, x=x, y=y, space=space,
             )
         ],
         config,
@@ -679,6 +765,26 @@ def screenshot(
     _run([action], config, verbose, fmt=format)
 
 
+@app.command()
+def activate(
+    window: WindowOption = None,
+    use: UseOption = None,
+    delay: DelayOption = None,
+    dry_run: DryRunOption = False,
+    verify: VerifyOption = False,
+    format: FormatOption = OutputFormat.TEXT,
+    verbose: VerboseOption = 0,
+) -> None:
+    """Bring a window forward and give it keyboard focus."""
+    config = _config(use, delay=delay, dry_run=dry_run, verify=verify, verbose=verbose)
+    _run(
+        [ActivateAction(window=_required(window, "--window"))],
+        config,
+        verbose,
+        fmt=format,
+    )
+
+
 # --- element commands ---------------------------------------------------------------------------
 
 
@@ -845,8 +951,12 @@ def batch(
     """Run a batch of actions over one connection."""
     raw = sys.stdin.read() if source == "-" else _read_file(source)
     try:
-        actions = ActionListAdapter.validate_json(raw)
-    except Exception as exc:
+        payload = normalise_action_names(json.loads(raw))
+        # Named before the union gets a chance: its own error is four hundred characters listing
+        # every variant except the one the caller should have written.
+        check_action_names(payload)
+        actions = ActionListAdapter.validate_python(payload)
+    except ValueError as exc:
         _say(
             "[red]error:[/red] {source} is not a valid action list: {exc}",
             source=source,
@@ -1070,13 +1180,21 @@ def _report_next_steps(kind: BackendKind, profile: str, needs_password: bool) ->
 
 
 @config_app.command("show")
-def config_show(use: UseOption = None) -> None:
+def config_show(
+    use: UseOption = None, format: FormatOption = OutputFormat.TEXT
+) -> None:
     """Print every resolved value, the layer it came from, and the variable that overrides it."""
     try:
         resolved = _config(use)
     except UseComputerError as exc:
         _fail(exc)
-    _emit(resolved.show())
+    payload = resolved.show()
+    if format is OutputFormat.JSON:
+        _emit(payload)
+        return
+    # This is the command named first when a profile misbehaves, so it is the worst one to answer
+    # with two kilobytes on one line.
+    _write(render.config(payload))
 
 
 # --- skill -------------------------------------------------------------------------------------
@@ -1162,6 +1280,7 @@ _COMMANDS = frozenset(
         "select",
         "set-value",
         "show-menu",
+        "activate",
         "batch",
         "config",
         "skill",