use-computer 0.0.27__tar.gz → 0.0.28__tar.gz

{use_computer-0.0.27 → use_computer-0.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: use-computer
-Version: 0.0.27
+Version: 0.0.28
 Summary: Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu
 Project-URL: Homepage, https://use.computer
 Project-URL: Documentation, https://api.use.computer/docs
@@ -18,6 +18,29 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Requires-Dist: httpx>=0.27
+Provides-Extra: agents
+Requires-Dist: pillow>=10; extra == 'agents'
+Provides-Extra: agents-all
+Requires-Dist: anthropic>=0.86; extra == 'agents-all'
+Requires-Dist: google-genai>=1; extra == 'agents-all'
+Requires-Dist: litellm>=1; extra == 'agents-all'
+Requires-Dist: openai>=1; extra == 'agents-all'
+Requires-Dist: pillow>=10; extra == 'agents-all'
+Provides-Extra: agents-anthropic
+Requires-Dist: anthropic>=0.86; extra == 'agents-anthropic'
+Requires-Dist: pillow>=10; extra == 'agents-anthropic'
+Provides-Extra: agents-gemini
+Requires-Dist: google-genai>=1; extra == 'agents-gemini'
+Requires-Dist: pillow>=10; extra == 'agents-gemini'
+Provides-Extra: agents-litellm
+Requires-Dist: litellm>=1; extra == 'agents-litellm'
+Requires-Dist: pillow>=10; extra == 'agents-litellm'
+Provides-Extra: agents-openai
+Requires-Dist: openai>=1; extra == 'agents-openai'
+Requires-Dist: pillow>=10; extra == 'agents-openai'
+Provides-Extra: harbor
+Requires-Dist: pyyaml>=6; extra == 'harbor'
+Requires-Dist: tomli>=2; extra == 'harbor'
 Description-Content-Type: text/markdown
 
 # use-computer Python SDK
@@ -29,6 +52,14 @@ pip install use-computer
 export USE_COMPUTER_API_KEY=mk_live_...
 ```
 
+Optional agent and Harbor integrations are installed explicitly:
+
+```bash
+pip install "use-computer[agents-openai]"     # or agents-anthropic / agents-gemini / agents-litellm / agents-all
+pip install "use-computer[harbor]"            # SDK Harbor adapter deps only; install Harbor separately until it is published
+pip install "use-computer[harbor,agents-all]" # Harbor adapter plus all provider deps
+```
+
 ```python
 from use_computer import Computer, SandboxType, SimulatorFamily
 

{use_computer-0.0.27 → use_computer-0.0.28}/README.md

@@ -7,6 +7,14 @@ pip install use-computer
 export USE_COMPUTER_API_KEY=mk_live_...
 ```
 
+Optional agent and Harbor integrations are installed explicitly:
+
+```bash
+pip install "use-computer[agents-openai]"     # or agents-anthropic / agents-gemini / agents-litellm / agents-all
+pip install "use-computer[harbor]"            # SDK Harbor adapter deps only; install Harbor separately until it is published
+pip install "use-computer[harbor,agents-all]" # Harbor adapter plus all provider deps
+```
+
 ```python
 from use_computer import Computer, SandboxType, SimulatorFamily
 

{use_computer-0.0.27 → use_computer-0.0.28}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer"
-version = "0.0.27"
+version = "0.0.28"
 description = "Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -23,11 +23,37 @@ classifiers = [
     "Typing :: Typed",
 ]
 
+[project.optional-dependencies]
+agents = ["Pillow>=10"]
+agents-openai = ["Pillow>=10", "openai>=1"]
+agents-anthropic = ["Pillow>=10", "anthropic>=0.86"]
+agents-litellm = ["Pillow>=10", "litellm>=1"]
+agents-gemini = ["Pillow>=10", "google-genai>=1"]
+agents-all = [
+    "Pillow>=10",
+    "openai>=1",
+    "anthropic>=0.86",
+    "litellm>=1",
+    "google-genai>=1",
+]
+harbor = [
+    "PyYAML>=6",
+    "tomli>=2",
+]
+
 [dependency-groups]
 dev = [
+    "anthropic>=0.86",
+    "google-genai>=1",
+    "harbor @ git+https://github.com/josancamon19/harbor.git@mirror ; python_version >= '3.12'",
+    "litellm>=1",
+    "openai>=1",
     "pre-commit>=4",
     "pytest>=9",
     "ruff>=0.15",
+    "Pillow>=10",
+    "PyYAML>=6",
+    "tomli>=2",
     "twine>=6",
     "ty>=0.0.34",
 ]

use_computer-0.0.28/use_computer/agents/__init__.py

@@ -0,0 +1,58 @@
+"""Optional computer-use agents.
+
+Install extras for provider-backed agents:
+    use-computer[agents-openai], use-computer[agents-anthropic],
+    use-computer[agents-litellm], use-computer[agents-gemini], or
+    use-computer[agents-all].
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import Any
+
+from use_computer.agents.core import AgentRunOptions, AgentRunResult, AgentUsage
+
+_LAZY_EXPORTS = {
+    "AnthropicComputerAgent": (
+        "use_computer.agents.providers.anthropic",
+        "AnthropicComputerAgent",
+    ),
+    "AnthropicCUAAgent": ("use_computer.agents.providers.anthropic", "AnthropicCUAAgent"),
+    "DebugComputerAgent": ("use_computer.agents.debug.agent", "DebugComputerAgent"),
+    "DebugCUAAgent": ("use_computer.agents.debug.agent", "DebugCUAAgent"),
+    "GeminiComputerAgent": ("use_computer.agents.providers.gemini", "GeminiComputerAgent"),
+    "GeminiCUAAgent": ("use_computer.agents.providers.gemini", "GeminiCUAAgent"),
+    "GenericComputerAgent": ("use_computer.agents.providers.generic", "GenericComputerAgent"),
+    "GenericCUAAgent": ("use_computer.agents.providers.generic", "GenericCUAAgent"),
+    "IOSAgent": ("use_computer.agents.ios.agent", "IOSAgent"),
+    "IOSComputerAgent": ("use_computer.agents.ios.agent", "IOSComputerAgent"),
+    "OpenAIComputerAgent": ("use_computer.agents.providers.openai", "OpenAIComputerAgent"),
+    "OpenAICUAAgent": ("use_computer.agents.providers.openai", "OpenAICUAAgent"),
+}
+
+__all__ = [
+    "AgentRunOptions",
+    "AgentRunResult",
+    "AgentUsage",
+    *_LAZY_EXPORTS,
+]
+
+
+def __getattr__(name: str) -> Any:
+    target = _LAZY_EXPORTS.get(name)
+    if target is None:
+        raise AttributeError(name)
+    module_name, attr = target
+    try:
+        module = import_module(module_name)
+    except ModuleNotFoundError as exc:
+        raise ModuleNotFoundError(
+            f"{name} requires an optional agent extra. Install one of "
+            "`use-computer[agents]`, `use-computer[agents-openai]`, "
+            "`use-computer[agents-anthropic]`, `use-computer[agents-litellm]`, "
+            "`use-computer[agents-gemini]`, or `use-computer[agents-all]`."
+        ) from exc
+    value = getattr(module, attr)
+    globals()[name] = value
+    return value

use_computer-0.0.28/use_computer/agents/base/__init__.py

@@ -0,0 +1,31 @@
+"""Public base-agent surface for CUA agents."""
+
+from use_computer.agents.base.actions import _annotate_click, execute_action
+from use_computer.agents.base.agent import BaseCUAAgent
+from use_computer.agents.base.desktop import (
+    DesktopProfile,
+    OpenAIComputerEnvironment,
+    desktop_profile,
+)
+from use_computer.agents.base.prompts import build_system_prompt, load_prompt
+from use_computer.agents.base.sandbox import ComputerSandbox
+from use_computer.agents.base.screenshots import (
+    scale_screenshot_for_model,
+    screenshot_cap_for_model,
+)
+from use_computer.agents.base.trajectory import write_trajectory
+
+__all__ = [
+    "BaseCUAAgent",
+    "ComputerSandbox",
+    "DesktopProfile",
+    "OpenAIComputerEnvironment",
+    "_annotate_click",
+    "build_system_prompt",
+    "desktop_profile",
+    "execute_action",
+    "load_prompt",
+    "scale_screenshot_for_model",
+    "screenshot_cap_for_model",
+    "write_trajectory",
+]

use_computer-0.0.28/use_computer/agents/base/actions.py

@@ -0,0 +1,234 @@
+"""Desktop action execution for CUA agents."""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import logging
+import time
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import httpx
+from PIL import Image, ImageDraw
+
+from use_computer.agents.base.desktop import normalize_key
+from use_computer.agents.base.sandbox import ComputerSandbox
+from use_computer.agents.types import DesktopAction, DesktopCommand, MouseButton
+
+_log = logging.getLogger("use_computer.agent")
+
+
+def _extract_coords(
+    action: dict[str, Any],
+) -> tuple[tuple[int, int] | None, tuple[int, int] | None]:
+    """Pull (point, secondary_point) from any action shape we use."""
+    point: tuple[int, int] | None = None
+    secondary: tuple[int, int] | None = None
+    if "coordinate" in action and action["coordinate"]:
+        c = action["coordinate"]
+        point = (int(c[0]), int(c[1]))
+    if "start_coordinate" in action and action["start_coordinate"]:
+        s = action["start_coordinate"]
+        secondary = (int(s[0]), int(s[1]))
+    if point is None and "x" in action and "y" in action:
+        try:
+            point = (int(float(action["x"])), int(float(action["y"])))
+        except (TypeError, ValueError):
+            pass
+    if "to_x" in action and "to_y" in action:
+        try:
+            point = (int(float(action["to_x"])), int(float(action["to_y"])))
+        except (TypeError, ValueError):
+            pass
+    if "from_x" in action and "from_y" in action:
+        try:
+            secondary = (int(float(action["from_x"])), int(float(action["from_y"])))
+        except (TypeError, ValueError):
+            pass
+    return point, secondary
+
+
+def _annotate_click(png_bytes: bytes, action: dict[str, Any]) -> bytes:
+    """Draw a red marker at the click/move/scroll/drag coordinate."""
+    point, start = _extract_coords(action)
+    if point is None and start is None:
+        return png_bytes
+    try:
+        img = Image.open(io.BytesIO(png_bytes)).convert("RGBA")
+        overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
+        draw = ImageDraw.Draw(overlay)
+        red = (255, 30, 30, 240)
+        radius = 5
+        if point and start:
+            draw.line([start, point], fill=red, width=2)
+            for px, py in (start, point):
+                draw.ellipse([px - radius, py - radius, px + radius, py + radius], fill=red)
+        else:
+            cx, cy = point or start  # type: ignore
+            draw.ellipse([cx - radius, cy - radius, cx + radius, cy + radius], fill=red)
+        out = Image.alpha_composite(img, overlay).convert("RGB")
+        buf = io.BytesIO()
+        out.save(buf, format="PNG")
+        return buf.getvalue()
+    except Exception as e:
+        _log.warning(f"annotate_click failed: {e}")
+        return png_bytes
+
+
+async def execute_action(
+    sandbox: ComputerSandbox,
+    action: dict[str, Any],
+    images_dir: Path,
+    step_idx: int,
+    key_aliases: Mapping[str, str] | None = None,
+) -> tuple[str, bytes | None, str | None]:
+    """Execute a computer-use action; returns (result_text, ss_bytes, ss_path)."""
+    command = DesktopCommand.from_mapping(action)
+    img_name = f"step_{step_idx:03d}.png"
+    t0 = time.monotonic()
+
+    if command.action is DesktopAction.SCREENSHOT:
+        ss = await _take_screenshot_with_recovery(sandbox, command.label)
+        if ss is None:
+            return "Screenshot failed after retries", None, None
+        (images_dir / img_name).write_bytes(ss)
+        _log.info(f"action screenshot: {time.monotonic() - t0:.2f}s, {len(ss)} bytes")
+        return "Screenshot taken", ss, f"images/{img_name}"
+
+    try:
+        if command.action in {DesktopAction.LEFT_CLICK, DesktopAction.CLICK}:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+            )
+        elif command.action is DesktopAction.RIGHT_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.RIGHT.value,
+            )
+        elif command.action is DesktopAction.DOUBLE_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+                double=True,
+            )
+        elif command.action is DesktopAction.TRIPLE_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+                double=True,
+            )
+            await asyncio.sleep(0.05)
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+            )
+        elif command.action is DesktopAction.MIDDLE_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.MIDDLE.value,
+            )
+        elif command.action is DesktopAction.TYPE:
+            await sandbox.keyboard.type(command.text)
+        elif command.action is DesktopAction.KEY:
+            key = normalize_key(command.key or command.text, key_aliases)
+            if "+" in key:
+                await sandbox.keyboard.hotkey(key)
+            else:
+                await sandbox.keyboard.press(key)
+        elif command.action is DesktopAction.SCROLL:
+            await sandbox.mouse.scroll(
+                command.point.x,
+                command.point.y,
+                command.direction.value,
+                command.amount,
+            )
+        elif command.action is DesktopAction.MOVE:
+            await sandbox.mouse.move(command.point.x, command.point.y)
+        elif command.action is DesktopAction.DRAG:
+            await sandbox.mouse.drag(
+                command.start_point.x,
+                command.start_point.y,
+                command.point.x,
+                command.point.y,
+            )
+        elif command.action is DesktopAction.WAIT:
+            await asyncio.sleep(command.duration)
+        else:
+            return f"Unknown action: {command.label}", None, None
+    except httpx.HTTPStatusError as e:
+        detail = e.response.text.strip().replace("\n", " ")[:240]
+        msg = f"Action '{command.label}' failed: HTTP {e.response.status_code} {detail}"
+        _log.warning(msg)
+        ss = await _take_screenshot_with_recovery(sandbox, command.label)
+        if ss is None:
+            return msg, None, None
+        (images_dir / img_name).write_bytes(ss)
+        return msg, ss, f"images/{img_name}"
+    except httpx.HTTPError as e:
+        msg = f"Action '{command.label}' failed: {type(e).__name__}: {e}"
+        _log.warning(msg)
+        ss = await _take_screenshot_with_recovery(sandbox, command.label)
+        if ss is None:
+            return msg, None, None
+        (images_dir / img_name).write_bytes(ss)
+        return msg, ss, f"images/{img_name}"
+
+    action_elapsed = time.monotonic() - t0
+    await asyncio.sleep(2)
+    ss = await _take_screenshot_with_recovery(sandbox, command.label)
+    if ss is None:
+        return (
+            f"Action '{command.label}' executed; screenshot failed after retries",
+            None,
+            None,
+        )
+    annotated = _annotate_click(ss, action)
+    (images_dir / img_name).write_bytes(annotated)
+    total_elapsed = time.monotonic() - t0
+    screenshot_elapsed = total_elapsed - action_elapsed
+    _log.info(
+        f"action {command.label}: {action_elapsed:.2f}s + "
+        f"screenshot {screenshot_elapsed:.2f}s = {total_elapsed:.2f}s total"
+    )
+    return f"Action '{command.label}' executed", ss, f"images/{img_name}"
+
+
+async def _take_screenshot_with_recovery(
+    sandbox: ComputerSandbox, action_type: str
+) -> bytes | None:
+    attempts = 4
+    for attempt in range(1, attempts + 1):
+        try:
+            return await sandbox.screenshot.take_full_screen()
+        except httpx.HTTPStatusError as e:
+            if attempt == attempts:
+                _log.warning(
+                    f"screenshot after {action_type} failed after retries: "
+                    f"HTTP {e.response.status_code} {e.response.text[:160]}"
+                )
+                return None
+            _log.warning(
+                f"screenshot after {action_type} failed: HTTP {e.response.status_code}; "
+                f"retrying in 10s ({attempt}/{attempts - 1})"
+            )
+            await asyncio.sleep(10)
+        except httpx.HTTPError as e:
+            if attempt == attempts:
+                _log.warning(
+                    f"screenshot after {action_type} failed after retries: {type(e).__name__}: {e}"
+                )
+                return None
+            _log.warning(
+                f"screenshot after {action_type} failed: {type(e).__name__}; "
+                f"retrying in 10s ({attempt}/{attempts - 1})"
+            )
+            await asyncio.sleep(10)