use-computer-cli 0.1.0__tar.gz

use_computer_cli-0.1.0/.gitignore

@@ -0,0 +1,47 @@
+# --- sdd -------------------------------------------------------------------
+# config.yaml carries the sdd remote api-key. This repository is public, so it
+# stays out. If you want the project description versioned, commit this file
+# only after moving the key elsewhere.
+.sdd/config.yaml
+
+# --- local project config (use-computer's own .use-computer/) --------------
+# This is a scratch config for driving the tool during development, not part of
+# the project. It carries allow-local, which is a per-machine opt-in and must
+# never arrive pre-enabled for whoever clones this.
+.use-computer/
+.env
+.env.*
+!.env.example
+
+# --- Python ----------------------------------------------------------------
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+*.egg
+build/
+dist/
+.eggs/
+.venv/
+venv/
+env/
+uv.lock
+
+# --- Tooling ---------------------------------------------------------------
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+htmlcov/
+
+# --- Node (sdd is an npm CLI) ---------------------------------------------
+node_modules/
+
+# --- Screenshots captured while driving the tool by hand -------------------
+*.png
+!product/**/*.png
+
+# --- Editor / OS -----------------------------------------------------------
+.DS_Store
+.vscode/
+.idea/

use_computer_cli-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bruno Fortunato
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

use_computer_cli-0.1.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.5
+Name: use-computer-cli
+Version: 0.1.0
+Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
+Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
+Author: Bruno Fortunato
+License: MIT License
+        
+        Copyright (c) 2026 Bruno Fortunato
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agent,automation,computer-use,screenshot,vnc
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pillow>=10
+Requires-Dist: pydantic-settings>=2.2
+Requires-Dist: pydantic>=2.7
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: rich>=13
+Requires-Dist: tomli>=2.0; python_version < '3.11'
+Requires-Dist: typer>=0.16
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest-mock>=3.12; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: local
+Requires-Dist: mss>=10; extra == 'local'
+Requires-Dist: pynput>=1.8; extra == 'local'
+Provides-Extra: vnc
+Requires-Dist: vncdotool>=1.3; extra == 'vnc'
+Description-Content-Type: text/markdown
+
+# use-computer
+
+Executes input on a screen for computer-use agents: move, click, double-click, right-click,
+drag, scroll, type text, press key combinations, and capture a screenshot.
+
+`use-computer` is the **acting** half of a pair. [ui-locator](https://github.com/applica-software-guru/ui-locator)
+answers *where* the Invia button is and returns pixel coordinates; `use-computer` performs the
+click there. Both are driven by another AI agent through a CLI that emits JSON on stdout and
+diagnostics on stderr, with a Python API underneath.
+
+## Install
+
+```bash
+pip install use-computer-cli            # no backend
+pip install "use-computer-cli[local]"   # drive this machine's display (pynput + mss)
+pip install "use-computer-cli[vnc]"     # drive a remote framebuffer over RFB (vncdotool)
+```
+
+Backends are optional extras, imported lazily, so the package installs without them.
+
+The distribution is `use-computer-cli` because `use-computer` is taken on PyPI by an unrelated
+project. The command it installs is `use-computer`, and the package it imports is `use_computer`.
+
+## Use
+
+```bash
+use-computer click --x 120 --y 340 --use staging
+use-computer type --text "hello" --use staging
+use-computer key ctrl+s --use staging
+use-computer screenshot --use laptop --out shot.png
+
+# a batch runs over one connection -- the default command, so `batch` may be omitted
+echo '[{"action":"click","x":120,"y":340},{"action":"key","combo":"enter"}]' \
+  | use-computer - --use staging --verify
+```
+
+stdout is one JSON object per run; every diagnostic goes to stderr. Exit codes: `0` success,
+`1` failure, `2` bad usage.
+
+## Three problems it solves
+
+- **Coordinate spaces.** A screenshot on a HiDPI display is larger than the space the OS clicks
+  in. Every coordinate carries its space, `use-computer` scales between them, and it refuses to
+  guess when the ratio is unknown.
+- **Setup cost.** Opening a VNC connection dominates a single action, so one run performs a
+  batch of actions over one connection.
+- **Blind actuation.** A click that lands on nothing looks exactly like a click that worked, so
+  `--verify` compares the screen before and after and reports whether it changed.
+
+## Configure
+
+```bash
+use-computer config init                                  # asks, then proves it works
+use-computer config init --backend vnc --host 10.0.0.5    # doesn't ask
+use-computer config init --backend local --allow-local
+```
+
+`config init` writes the file below, then opens the backend it just configured and reports the
+screen geometry and scale — so a coordinate space whose ratio cannot be derived surfaces at setup
+rather than at the first click that lands in the wrong place.
+
+It writes `.use-computer/config.toml` at the project root (found by walking up, the way git finds
+its own):
+
+```toml
+default-profile = "laptop"
+delay = 0.1
+
+[profiles.laptop]
+backend = "local"
+allow-local = true
+
+[profiles.staging]
+backend = "vnc"
+host = "10.0.0.5"
+port = 5900
+```
+
+Secrets go in `.use-computer/.env`, which is not committed. `use-computer config show` prints
+every resolved value, the layer it came from and the variable that would override it.
+
+## The agent skill
+
+Instructions for the calling agent ship inside the package and are installed from it, so they
+always match the installed version:
+
+```bash
+use-computer skill install --scope project
+```
+
+## Documentation
+
+This package is developed with [SDD](https://github.com/applica-software-guru/sdd). The specs
+it implements live in `product/` and `system/` at the repository root.

use_computer_cli-0.1.0/README.md

@@ -0,0 +1,94 @@
+# use-computer
+
+Executes input on a screen for computer-use agents: move, click, double-click, right-click,
+drag, scroll, type text, press key combinations, and capture a screenshot.
+
+`use-computer` is the **acting** half of a pair. [ui-locator](https://github.com/applica-software-guru/ui-locator)
+answers *where* the Invia button is and returns pixel coordinates; `use-computer` performs the
+click there. Both are driven by another AI agent through a CLI that emits JSON on stdout and
+diagnostics on stderr, with a Python API underneath.
+
+## Install
+
+```bash
+pip install use-computer-cli            # no backend
+pip install "use-computer-cli[local]"   # drive this machine's display (pynput + mss)
+pip install "use-computer-cli[vnc]"     # drive a remote framebuffer over RFB (vncdotool)
+```
+
+Backends are optional extras, imported lazily, so the package installs without them.
+
+The distribution is `use-computer-cli` because `use-computer` is taken on PyPI by an unrelated
+project. The command it installs is `use-computer`, and the package it imports is `use_computer`.
+
+## Use
+
+```bash
+use-computer click --x 120 --y 340 --use staging
+use-computer type --text "hello" --use staging
+use-computer key ctrl+s --use staging
+use-computer screenshot --use laptop --out shot.png
+
+# a batch runs over one connection -- the default command, so `batch` may be omitted
+echo '[{"action":"click","x":120,"y":340},{"action":"key","combo":"enter"}]' \
+  | use-computer - --use staging --verify
+```
+
+stdout is one JSON object per run; every diagnostic goes to stderr. Exit codes: `0` success,
+`1` failure, `2` bad usage.
+
+## Three problems it solves
+
+- **Coordinate spaces.** A screenshot on a HiDPI display is larger than the space the OS clicks
+  in. Every coordinate carries its space, `use-computer` scales between them, and it refuses to
+  guess when the ratio is unknown.
+- **Setup cost.** Opening a VNC connection dominates a single action, so one run performs a
+  batch of actions over one connection.
+- **Blind actuation.** A click that lands on nothing looks exactly like a click that worked, so
+  `--verify` compares the screen before and after and reports whether it changed.
+
+## Configure
+
+```bash
+use-computer config init                                  # asks, then proves it works
+use-computer config init --backend vnc --host 10.0.0.5    # doesn't ask
+use-computer config init --backend local --allow-local
+```
+
+`config init` writes the file below, then opens the backend it just configured and reports the
+screen geometry and scale — so a coordinate space whose ratio cannot be derived surfaces at setup
+rather than at the first click that lands in the wrong place.
+
+It writes `.use-computer/config.toml` at the project root (found by walking up, the way git finds
+its own):
+
+```toml
+default-profile = "laptop"
+delay = 0.1
+
+[profiles.laptop]
+backend = "local"
+allow-local = true
+
+[profiles.staging]
+backend = "vnc"
+host = "10.0.0.5"
+port = 5900
+```
+
+Secrets go in `.use-computer/.env`, which is not committed. `use-computer config show` prints
+every resolved value, the layer it came from and the variable that would override it.
+
+## The agent skill
+
+Instructions for the calling agent ship inside the package and are installed from it, so they
+always match the installed version:
+
+```bash
+use-computer skill install --scope project
+```
+
+## Documentation
+
+This package is developed with [SDD](https://github.com/applica-software-guru/sdd). The specs
+it implements live in `product/` and `system/` at the repository root.

use_computer_cli-0.1.0/pyproject.toml

@@ -0,0 +1,86 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "use-computer-cli"
+version = "0.1.0"
+description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+authors = [{ name = "Bruno Fortunato" }]
+keywords = ["computer-use", "automation", "vnc", "agent", "screenshot"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Typing :: Typed",
+]
+
+# Floors are the versions actually tested in CI. typer 0.12 is explicitly unsupported:
+# it predates the API this CLI relies on.
+dependencies = [
+    "pydantic>=2.7",
+    "pydantic-settings>=2.2",
+    "typer>=0.16",
+    "rich>=13",
+    "pillow>=10",
+    "python-dotenv>=1.0",
+    "tomli>=2.0; python_version < '3.11'",
+]
+
+[project.optional-dependencies]
+# Backends are optional extras, imported lazily, so the package installs without them.
+# pyautogui is deliberately not used: its last release (0.9.54) dates from 2023.
+local = ["pynput>=1.8", "mss>=10"]
+vnc = ["vncdotool>=1.3"]
+dev = [
+    "pytest>=8",
+    "pytest-mock>=3.12",
+    "ruff>=0.6",
+    "mypy>=1.11",
+]
+
+[project.scripts]
+use-computer = "use_computer.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/applica-software-guru/use-computer"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/use_computer"]
+
+[tool.hatch.build.targets.wheel.force-include]
+# The agent skill ships inside the wheel. A copy at the repository root would not.
+"src/use_computer/skill/SKILL.md" = "use_computer/skill/SKILL.md"
+
+[tool.hatch.build.targets.sdist]
+include = ["src/use_computer", "tests", "README.md", "LICENSE", "pyproject.toml"]
+
+[tool.ruff]
+line-length = 100
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "C4"]
+
+[tool.ruff.lint.per-file-ignores]
+# typer idiom: Option()/Argument() defaults are function calls in the signature.
+"src/use_computer/cli.py" = ["B008"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+files = ["src", "tests"]
+
+[[tool.mypy.overrides]]
+module = ["pynput.*", "mss.*", "vncdotool.*", "ApplicationServices.*", "Quartz.*", "tomli"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

use_computer_cli-0.1.0/src/use_computer/__init__.py

@@ -0,0 +1,93 @@
+"""use-computer: execute input on a screen for computer-use agents.
+
+The acting half of a pair. ui-locator answers where; this performs the action there.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from use_computer.actions import (
+    Action,
+    ClickAction,
+    DoubleClickAction,
+    DragAction,
+    KeyAction,
+    MouseButton,
+    MoveAction,
+    RightClickAction,
+    ScreenshotAction,
+    ScrollAction,
+    ScrollDirection,
+    TypeAction,
+)
+from use_computer.backends import Backend, create_backend
+from use_computer.compare import ChangeReport, Screenshot, compare
+from use_computer.config import BackendProfile, ResolvedConfig, Settings
+from use_computer.config import load as load_config
+from use_computer.coordinates import Coordinate, CoordinateSpace, ScreenInfo, convert
+from use_computer.errors import (
+    ActionFailedError,
+    BackendNotAvailableError,
+    ConfigError,
+    CoordinateSpaceError,
+    KeySyntaxError,
+    PermissionDeniedError,
+    UseComputerError,
+)
+from use_computer.keys import KeyCombo, parse_combo
+from use_computer.runner import ActionResult, ErrorInfo, RunResult, Session, run_actions
+
+#: The distribution name on PyPI, which differs from the import package: `use-computer` was
+#: already taken there. importlib.metadata is keyed by the distribution, so this is the name
+#: that must appear here.
+DISTRIBUTION = "use-computer-cli"
+
+try:
+    __version__ = version(DISTRIBUTION)
+except PackageNotFoundError:  # pragma: no cover - source checkout without an install
+    __version__ = "0.0.0"
+
+__all__ = [
+    "DISTRIBUTION",
+    "Action",
+    "ActionFailedError",
+    "ActionResult",
+    "Backend",
+    "BackendNotAvailableError",
+    "BackendProfile",
+    "ChangeReport",
+    "ClickAction",
+    "ConfigError",
+    "Coordinate",
+    "CoordinateSpace",
+    "CoordinateSpaceError",
+    "DoubleClickAction",
+    "DragAction",
+    "ErrorInfo",
+    "KeyAction",
+    "KeyCombo",
+    "KeySyntaxError",
+    "MouseButton",
+    "MoveAction",
+    "PermissionDeniedError",
+    "ResolvedConfig",
+    "RightClickAction",
+    "RunResult",
+    "ScreenInfo",
+    "Screenshot",
+    "ScreenshotAction",
+    "ScrollAction",
+    "ScrollDirection",
+    "Session",
+    "Settings",
+    "TypeAction",
+    "UseComputerError",
+    "__version__",
+    "compare",
+    "convert",
+    "create_backend",
+    "load_config",
+    "parse_combo",
+    "run_actions",
+]

use_computer_cli-0.1.0/src/use_computer/actions.py

@@ -0,0 +1,184 @@
+"""The action models, and the resolution that happens before a backend sees them.
+
+The action set is the whole surface of what use-computer does to a screen. Coordinate scaling
+and key parsing are applied here, so that everything crossing the backend boundary is already
+in actuation units and canonical key names.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
+
+from use_computer.coordinates import Coordinate, CoordinateSpace, ScreenInfo, convert
+from use_computer.keys import KeyCombo, canonical, parse_combo
+
+
+class MouseButton(str, Enum):
+    LEFT = "left"
+    RIGHT = "right"
+    MIDDLE = "middle"
+
+
+class ScrollDirection(str, Enum):
+    UP = "up"
+    DOWN = "down"
+    LEFT = "left"
+    RIGHT = "right"
+
+
+class BaseAction(BaseModel):
+    """Fields every action shares."""
+
+    delay: float | None = Field(default=None, ge=0, description="Seconds to wait afterwards.")
+    verify: bool = Field(default=False, description="Compare the screen before and after.")
+
+    @property
+    def target(self) -> Coordinate | None:
+        """The point this action acts on, or None if it has no coordinate."""
+        return None
+
+    @property
+    def origin(self) -> Coordinate | None:
+        """The point this action starts from -- only a drag has one."""
+        return None
+
+
+class _Positioned(BaseAction):
+    """An action that may carry a coordinate. Omitting it acts where the pointer already is."""
+
+    x: int | None = None
+    y: int | None = None
+    space: CoordinateSpace | None = None
+
+    @property
+    def target(self) -> Coordinate | None:
+        if self.x is None or self.y is None:
+            return None
+        space = self.space if self.space is not None else CoordinateSpace.SCREENSHOT
+        return Coordinate(x=self.x, y=self.y, space=space)
+
+
+class MoveAction(_Positioned):
+    action: Literal["move"] = "move"
+    x: int
+    y: int
+
+
+class ClickAction(_Positioned):
+    action: Literal["click"] = "click"
+    button: MouseButton = MouseButton.LEFT
+
+
+class DoubleClickAction(_Positioned):
+    action: Literal["double_click"] = "double_click"
+    button: MouseButton = MouseButton.LEFT
+
+
+class RightClickAction(_Positioned):
+    action: Literal["right_click"] = "right_click"
+
+
+class DragAction(BaseAction):
+    action: Literal["drag"] = "drag"
+    from_x: int
+    from_y: int
+    to_x: int
+    to_y: int
+    space: CoordinateSpace | None = None
+    button: MouseButton = MouseButton.LEFT
+
+    @property
+    def origin(self) -> Coordinate:
+        return Coordinate(x=self.from_x, y=self.from_y, space=self._space)
+
+    @property
+    def target(self) -> Coordinate:
+        return Coordinate(x=self.to_x, y=self.to_y, space=self._space)
+
+    @property
+    def _space(self) -> CoordinateSpace:
+        return self.space if self.space is not None else CoordinateSpace.SCREENSHOT
+
+
+class ScrollAction(_Positioned):
+    action: Literal["scroll"] = "scroll"
+    amount: int
+    direction: ScrollDirection = ScrollDirection.DOWN
+
+
+class TypeAction(BaseAction):
+    action: Literal["type"] = "type"
+    text: str
+    rate: float | None = Field(
+        default=None,
+        ge=0,
+        description="Seconds between keystrokes; None uses the configured rate.",
+    )
+
+
+class KeyAction(BaseAction):
+    action: Literal["key"] = "key"
+    combo: str
+
+    @field_validator("combo")
+    @classmethod
+    def _canonicalise(cls, value: str) -> str:
+        # Parse at construction: an unknown key name must fail before a connection is opened,
+        # and the canonical spelling is what appears in results and logs.
+        return canonical(value)
+
+    @property
+    def key_combo(self) -> KeyCombo:
+        return parse_combo(self.combo)
+
+
+class ScreenshotAction(BaseAction):
+    action: Literal["screenshot"] = "screenshot"
+    out: Path | None = None
+    base64: bool = False
+
+
+Action = Annotated[
+    MoveAction
+    | ClickAction
+    | DoubleClickAction
+    | RightClickAction
+    | DragAction
+    | ScrollAction
+    | TypeAction
+    | KeyAction
+    | ScreenshotAction,
+    Field(discriminator="action"),
+]
+
+#: Parses a batch file: a JSON array of action objects, discriminated on `action`.
+ActionListAdapter: TypeAdapter[list[Action]] = TypeAdapter(list[Action])
+
+#: Parses a single action object.
+ActionAdapter: TypeAdapter[Action] = TypeAdapter(Action)
+
+
+def with_default_space(action: Action, space: CoordinateSpace) -> Action:
+    """Fill in the coordinate space an action did not state, from configuration."""
+    if getattr(action, "space", "missing") is None:
+        return action.model_copy(update={"space": space})
+    return action
+
+
+def resolve(action: Action, screen: ScreenInfo) -> tuple[Coordinate | None, Coordinate | None]:
+    """Convert an action's coordinates into actuation units.
+
+    Returns ``(target, origin)``; either may be ``None`` when the action carries no coordinate.
+
+    Raises:
+        CoordinateSpaceError: when the scale needed for the conversion is unknown.
+    """
+    target = action.target
+    origin = action.origin
+    resolved_target = convert(target, CoordinateSpace.ACTUATION, screen) if target else None
+    resolved_origin = convert(origin, CoordinateSpace.ACTUATION, screen) if origin else None
+    return resolved_target, resolved_origin