use-computer 0.0.26__tar.gz → 0.0.28__tar.gz

use_computer-0.0.26/README.md → use_computer-0.0.28/PKG-INFO

@@ -1,3 +1,48 @@
+Metadata-Version: 2.4
+Name: use-computer
+Version: 0.0.28
+Summary: Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu
+Project-URL: Homepage, https://use.computer
+Project-URL: Documentation, https://api.use.computer/docs
+Project-URL: Repository, https://github.com/josancamon19/use-computer-python
+Author: use.computer
+Keywords: automation,computer-use,ios,macos,sandbox,ubuntu,vnc,windows
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Provides-Extra: agents
+Requires-Dist: pillow>=10; extra == 'agents'
+Provides-Extra: agents-all
+Requires-Dist: anthropic>=0.86; extra == 'agents-all'
+Requires-Dist: google-genai>=1; extra == 'agents-all'
+Requires-Dist: litellm>=1; extra == 'agents-all'
+Requires-Dist: openai>=1; extra == 'agents-all'
+Requires-Dist: pillow>=10; extra == 'agents-all'
+Provides-Extra: agents-anthropic
+Requires-Dist: anthropic>=0.86; extra == 'agents-anthropic'
+Requires-Dist: pillow>=10; extra == 'agents-anthropic'
+Provides-Extra: agents-gemini
+Requires-Dist: google-genai>=1; extra == 'agents-gemini'
+Requires-Dist: pillow>=10; extra == 'agents-gemini'
+Provides-Extra: agents-litellm
+Requires-Dist: litellm>=1; extra == 'agents-litellm'
+Requires-Dist: pillow>=10; extra == 'agents-litellm'
+Provides-Extra: agents-openai
+Requires-Dist: openai>=1; extra == 'agents-openai'
+Requires-Dist: pillow>=10; extra == 'agents-openai'
+Provides-Extra: harbor
+Requires-Dist: pyyaml>=6; extra == 'harbor'
+Requires-Dist: tomli>=2; extra == 'harbor'
+Description-Content-Type: text/markdown
+
 # use-computer Python SDK
 
 Python client for [use.computer](https://use.computer) — rent dedicated VMs across macOS, iOS/visionOS/tvOS simulators, Windows, and Ubuntu built for computer-use agents.
@@ -7,6 +52,14 @@ pip install use-computer
 export USE_COMPUTER_API_KEY=mk_live_...
 ```
 
+Optional agent and Harbor integrations are installed explicitly:
+
+```bash
+pip install "use-computer[agents-openai]"     # or agents-anthropic / agents-gemini / agents-litellm / agents-all
+pip install "use-computer[harbor]"            # SDK Harbor adapter deps only; install Harbor separately until it is published
+pip install "use-computer[harbor,agents-all]" # Harbor adapter plus all provider deps
+```
+
 ```python
 from use_computer import Computer, SandboxType, SimulatorFamily
 
@@ -38,18 +91,62 @@ recording/file surface as macOS; exec runs in-guest via PowerShell/cmd
 (`win.run` / `win.shell`) instead of SSH. `AsyncWindowsSandbox` mirrors it.
 See [docs.use.computer/docs/windows](https://docs.use.computer/docs/windows).
 
+Selectable Windows images are exposed by `Computer().platforms()["windows"]`.
+Prefer `version="windows-11"` plus `resources={"cpus": 4, "memory_mb": 4096,
+"disk_gb": 40}` to select a resource profile. Explicit image IDs such as
+`windows-11-4c4g40g` still work. The image metadata includes CPU/RAM/disk and
+display size.
+
 ### Ubuntu (Beta)
 
 ```python
 with Computer().create(type="ubuntu", version="ubuntu-24.04") as ubuntu:
     print(ubuntu.run("uname -a").stdout)  # bash exec (no SSH)
+    print(ubuntu.display.get_info())
     ubuntu.keyboard.type("hello")
     ubuntu.screenshot.take_full_screen()
 ```
 
 Ubuntu sandboxes are **Beta** (admin-only). They use the same KVM/noVNC backend
-as Windows; pass `version="ubuntu-24.04"` to select a specific image. Exec
-defaults to bash and the macOS accessibility tree is not available yet.
+as Windows; pass `version="ubuntu-24.04"` plus `resources` to select CPU/RAM/disk.
+
+Selectable Ubuntu images are exposed by `Computer().platforms()["ubuntu"]`.
+Current examples include `ubuntu-24.04`, `ubuntu-24.04-4c4g80g`,
+`ubuntu-24.04-4c4g40g`, and `ubuntu-24.04-2c4g40g`.
+
+```python
+client = Computer()
+platforms = client.platforms()
+for image in platforms["ubuntu"]["images"]:
+    print(image["version"], image["resources"], image["display"])
+
+with client.create(
+    type="ubuntu",
+    version="ubuntu-24.04",
+    resources={"cpus": 2, "memory_mb": 4096, "disk_gb": 40},
+) as ubuntu:
+    print(ubuntu.display.get_info())
+```
+
+### Snapshots (Beta)
+
+Windows and Ubuntu snapshots preserve disk + RAM state, including open apps and
+running processes. Seed a VM once, snapshot it, then create new sandboxes from
+that snapshot version.
+
+```python
+client = Computer()
+
+with client.create(type="ubuntu", version="ubuntu-24.04") as ubuntu:
+    ubuntu.run("google-chrome-stable --no-first-run https://example.com >/tmp/chrome.log 2>&1 &")
+    snapshot = ubuntu.snapshot("chrome-seeded-ubuntu")
+
+with client.create(type="ubuntu", snapshot=snapshot.version) as seeded:
+    print(seeded.run("pgrep -af 'chrome|chromium' | head").stdout)
+```
+
+Use `client.snapshots("ubuntu")` or `client.snapshots("windows")` to list saved
+snapshot versions.
 
 Full DSL reference: [docs.use.computer/docs/sdk](https://docs.use.computer/docs/sdk)
 
@@ -85,6 +182,8 @@ visionOS yet. Use it for read-only flows for now.
 | [`examples/_6_hello_tvos.py`](examples/_6_hello_tvos.py)       | tvOS: pick TV family + drive the Apple Remote  |
 | [`examples/_7_hello_windows.py`](examples/_7_hello_windows.py) | create Windows → run PowerShell → screenshot   |
 | [`examples/_8_hello_ubuntu.py`](examples/_8_hello_ubuntu.py)   | create Ubuntu → run bash → screenshot          |
+| [`examples/_9_seeding.py`](examples/_9_seeding.py)             | typed setup: files, hosts, open URLs/apps      |
+| [`examples/_10_snapshots.py`](examples/_10_snapshots.py)       | snapshot seeded Ubuntu/Windows desktop state   |
 
 For agent loops and evals: [use-computer-cookbook](https://github.com/josancamon19/use-computer-cookbook).
 

use_computer-0.0.26/PKG-INFO → use_computer-0.0.28/README.md

@@ -1,25 +1,3 @@
-Metadata-Version: 2.4
-Name: use-computer
-Version: 0.0.26
-Summary: Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu
-Project-URL: Homepage, https://use.computer
-Project-URL: Documentation, https://api.use.computer/docs
-Project-URL: Repository, https://github.com/josancamon19/use-computer-python
-Author: use.computer
-Keywords: automation,computer-use,ios,macos,sandbox,ubuntu,vnc,windows
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Typing :: Typed
-Requires-Python: >=3.10
-Requires-Dist: httpx>=0.27
-Description-Content-Type: text/markdown
-
 # use-computer Python SDK
 
 Python client for [use.computer](https://use.computer) — rent dedicated VMs across macOS, iOS/visionOS/tvOS simulators, Windows, and Ubuntu built for computer-use agents.
@@ -29,6 +7,14 @@ pip install use-computer
 export USE_COMPUTER_API_KEY=mk_live_...
 ```
 
+Optional agent and Harbor integrations are installed explicitly:
+
+```bash
+pip install "use-computer[agents-openai]"     # or agents-anthropic / agents-gemini / agents-litellm / agents-all
+pip install "use-computer[harbor]"            # SDK Harbor adapter deps only; install Harbor separately until it is published
+pip install "use-computer[harbor,agents-all]" # Harbor adapter plus all provider deps
+```
+
 ```python
 from use_computer import Computer, SandboxType, SimulatorFamily
 
@@ -60,18 +46,62 @@ recording/file surface as macOS; exec runs in-guest via PowerShell/cmd
 (`win.run` / `win.shell`) instead of SSH. `AsyncWindowsSandbox` mirrors it.
 See [docs.use.computer/docs/windows](https://docs.use.computer/docs/windows).
 
+Selectable Windows images are exposed by `Computer().platforms()["windows"]`.
+Prefer `version="windows-11"` plus `resources={"cpus": 4, "memory_mb": 4096,
+"disk_gb": 40}` to select a resource profile. Explicit image IDs such as
+`windows-11-4c4g40g` still work. The image metadata includes CPU/RAM/disk and
+display size.
+
 ### Ubuntu (Beta)
 
 ```python
 with Computer().create(type="ubuntu", version="ubuntu-24.04") as ubuntu:
     print(ubuntu.run("uname -a").stdout)  # bash exec (no SSH)
+    print(ubuntu.display.get_info())
     ubuntu.keyboard.type("hello")
     ubuntu.screenshot.take_full_screen()
 ```
 
 Ubuntu sandboxes are **Beta** (admin-only). They use the same KVM/noVNC backend
-as Windows; pass `version="ubuntu-24.04"` to select a specific image. Exec
-defaults to bash and the macOS accessibility tree is not available yet.
+as Windows; pass `version="ubuntu-24.04"` plus `resources` to select CPU/RAM/disk.
+
+Selectable Ubuntu images are exposed by `Computer().platforms()["ubuntu"]`.
+Current examples include `ubuntu-24.04`, `ubuntu-24.04-4c4g80g`,
+`ubuntu-24.04-4c4g40g`, and `ubuntu-24.04-2c4g40g`.
+
+```python
+client = Computer()
+platforms = client.platforms()
+for image in platforms["ubuntu"]["images"]:
+    print(image["version"], image["resources"], image["display"])
+
+with client.create(
+    type="ubuntu",
+    version="ubuntu-24.04",
+    resources={"cpus": 2, "memory_mb": 4096, "disk_gb": 40},
+) as ubuntu:
+    print(ubuntu.display.get_info())
+```
+
+### Snapshots (Beta)
+
+Windows and Ubuntu snapshots preserve disk + RAM state, including open apps and
+running processes. Seed a VM once, snapshot it, then create new sandboxes from
+that snapshot version.
+
+```python
+client = Computer()
+
+with client.create(type="ubuntu", version="ubuntu-24.04") as ubuntu:
+    ubuntu.run("google-chrome-stable --no-first-run https://example.com >/tmp/chrome.log 2>&1 &")
+    snapshot = ubuntu.snapshot("chrome-seeded-ubuntu")
+
+with client.create(type="ubuntu", snapshot=snapshot.version) as seeded:
+    print(seeded.run("pgrep -af 'chrome|chromium' | head").stdout)
+```
+
+Use `client.snapshots("ubuntu")` or `client.snapshots("windows")` to list saved
+snapshot versions.
 
 Full DSL reference: [docs.use.computer/docs/sdk](https://docs.use.computer/docs/sdk)
 
@@ -107,6 +137,8 @@ visionOS yet. Use it for read-only flows for now.
 | [`examples/_6_hello_tvos.py`](examples/_6_hello_tvos.py)       | tvOS: pick TV family + drive the Apple Remote  |
 | [`examples/_7_hello_windows.py`](examples/_7_hello_windows.py) | create Windows → run PowerShell → screenshot   |
 | [`examples/_8_hello_ubuntu.py`](examples/_8_hello_ubuntu.py)   | create Ubuntu → run bash → screenshot          |
+| [`examples/_9_seeding.py`](examples/_9_seeding.py)             | typed setup: files, hosts, open URLs/apps      |
+| [`examples/_10_snapshots.py`](examples/_10_snapshots.py)       | snapshot seeded Ubuntu/Windows desktop state   |
 
 For agent loops and evals: [use-computer-cookbook](https://github.com/josancamon19/use-computer-cookbook).
 

{use_computer-0.0.26 → use_computer-0.0.28}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer"
-version = "0.0.26"
+version = "0.0.28"
 description = "Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -23,11 +23,37 @@ classifiers = [
     "Typing :: Typed",
 ]
 
+[project.optional-dependencies]
+agents = ["Pillow>=10"]
+agents-openai = ["Pillow>=10", "openai>=1"]
+agents-anthropic = ["Pillow>=10", "anthropic>=0.86"]
+agents-litellm = ["Pillow>=10", "litellm>=1"]
+agents-gemini = ["Pillow>=10", "google-genai>=1"]
+agents-all = [
+    "Pillow>=10",
+    "openai>=1",
+    "anthropic>=0.86",
+    "litellm>=1",
+    "google-genai>=1",
+]
+harbor = [
+    "PyYAML>=6",
+    "tomli>=2",
+]
+
 [dependency-groups]
 dev = [
+    "anthropic>=0.86",
+    "google-genai>=1",
+    "harbor @ git+https://github.com/josancamon19/harbor.git@mirror ; python_version >= '3.12'",
+    "litellm>=1",
+    "openai>=1",
     "pre-commit>=4",
     "pytest>=9",
     "ruff>=0.15",
+    "Pillow>=10",
+    "PyYAML>=6",
+    "tomli>=2",
     "twine>=6",
     "ty>=0.0.34",
 ]
@@ -48,9 +74,12 @@ exclude = [
     "/.venv",
     "/dist",
     "/results",
-    "/use_computer/test_*.py",
+    "/tests",
 ]
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
 [tool.ruff]
 target-version = "py310"
 line-length = 100

use_computer-0.0.28/use_computer/accessibility.py

@@ -0,0 +1,3 @@
+from use_computer.resources.accessibility import Accessibility, AsyncAccessibility
+
+__all__ = ["Accessibility", "AsyncAccessibility"]

use_computer-0.0.28/use_computer/agents/__init__.py

@@ -0,0 +1,58 @@
+"""Optional computer-use agents.
+
+Install extras for provider-backed agents:
+    use-computer[agents-openai], use-computer[agents-anthropic],
+    use-computer[agents-litellm], use-computer[agents-gemini], or
+    use-computer[agents-all].
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import Any
+
+from use_computer.agents.core import AgentRunOptions, AgentRunResult, AgentUsage
+
+_LAZY_EXPORTS = {
+    "AnthropicComputerAgent": (
+        "use_computer.agents.providers.anthropic",
+        "AnthropicComputerAgent",
+    ),
+    "AnthropicCUAAgent": ("use_computer.agents.providers.anthropic", "AnthropicCUAAgent"),
+    "DebugComputerAgent": ("use_computer.agents.debug.agent", "DebugComputerAgent"),
+    "DebugCUAAgent": ("use_computer.agents.debug.agent", "DebugCUAAgent"),
+    "GeminiComputerAgent": ("use_computer.agents.providers.gemini", "GeminiComputerAgent"),
+    "GeminiCUAAgent": ("use_computer.agents.providers.gemini", "GeminiCUAAgent"),
+    "GenericComputerAgent": ("use_computer.agents.providers.generic", "GenericComputerAgent"),
+    "GenericCUAAgent": ("use_computer.agents.providers.generic", "GenericCUAAgent"),
+    "IOSAgent": ("use_computer.agents.ios.agent", "IOSAgent"),
+    "IOSComputerAgent": ("use_computer.agents.ios.agent", "IOSComputerAgent"),
+    "OpenAIComputerAgent": ("use_computer.agents.providers.openai", "OpenAIComputerAgent"),
+    "OpenAICUAAgent": ("use_computer.agents.providers.openai", "OpenAICUAAgent"),
+}
+
+__all__ = [
+    "AgentRunOptions",
+    "AgentRunResult",
+    "AgentUsage",
+    *_LAZY_EXPORTS,
+]
+
+
+def __getattr__(name: str) -> Any:
+    target = _LAZY_EXPORTS.get(name)
+    if target is None:
+        raise AttributeError(name)
+    module_name, attr = target
+    try:
+        module = import_module(module_name)
+    except ModuleNotFoundError as exc:
+        raise ModuleNotFoundError(
+            f"{name} requires an optional agent extra. Install one of "
+            "`use-computer[agents]`, `use-computer[agents-openai]`, "
+            "`use-computer[agents-anthropic]`, `use-computer[agents-litellm]`, "
+            "`use-computer[agents-gemini]`, or `use-computer[agents-all]`."
+        ) from exc
+    value = getattr(module, attr)
+    globals()[name] = value
+    return value

use_computer-0.0.28/use_computer/agents/base/__init__.py

@@ -0,0 +1,31 @@
+"""Public base-agent surface for CUA agents."""
+
+from use_computer.agents.base.actions import _annotate_click, execute_action
+from use_computer.agents.base.agent import BaseCUAAgent
+from use_computer.agents.base.desktop import (
+    DesktopProfile,
+    OpenAIComputerEnvironment,
+    desktop_profile,
+)
+from use_computer.agents.base.prompts import build_system_prompt, load_prompt
+from use_computer.agents.base.sandbox import ComputerSandbox
+from use_computer.agents.base.screenshots import (
+    scale_screenshot_for_model,
+    screenshot_cap_for_model,
+)
+from use_computer.agents.base.trajectory import write_trajectory
+
+__all__ = [
+    "BaseCUAAgent",
+    "ComputerSandbox",
+    "DesktopProfile",
+    "OpenAIComputerEnvironment",
+    "_annotate_click",
+    "build_system_prompt",
+    "desktop_profile",
+    "execute_action",
+    "load_prompt",
+    "scale_screenshot_for_model",
+    "screenshot_cap_for_model",
+    "write_trajectory",
+]

use_computer-0.0.28/use_computer/agents/base/actions.py

@@ -0,0 +1,234 @@
+"""Desktop action execution for CUA agents."""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import logging
+import time
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import httpx
+from PIL import Image, ImageDraw
+
+from use_computer.agents.base.desktop import normalize_key
+from use_computer.agents.base.sandbox import ComputerSandbox
+from use_computer.agents.types import DesktopAction, DesktopCommand, MouseButton
+
+_log = logging.getLogger("use_computer.agent")
+
+
+def _extract_coords(
+    action: dict[str, Any],
+) -> tuple[tuple[int, int] | None, tuple[int, int] | None]:
+    """Pull (point, secondary_point) from any action shape we use."""
+    point: tuple[int, int] | None = None
+    secondary: tuple[int, int] | None = None
+    if "coordinate" in action and action["coordinate"]:
+        c = action["coordinate"]
+        point = (int(c[0]), int(c[1]))
+    if "start_coordinate" in action and action["start_coordinate"]:
+        s = action["start_coordinate"]
+        secondary = (int(s[0]), int(s[1]))
+    if point is None and "x" in action and "y" in action:
+        try:
+            point = (int(float(action["x"])), int(float(action["y"])))
+        except (TypeError, ValueError):
+            pass
+    if "to_x" in action and "to_y" in action:
+        try:
+            point = (int(float(action["to_x"])), int(float(action["to_y"])))
+        except (TypeError, ValueError):
+            pass
+    if "from_x" in action and "from_y" in action:
+        try:
+            secondary = (int(float(action["from_x"])), int(float(action["from_y"])))
+        except (TypeError, ValueError):
+            pass
+    return point, secondary
+
+
+def _annotate_click(png_bytes: bytes, action: dict[str, Any]) -> bytes:
+    """Draw a red marker at the click/move/scroll/drag coordinate."""
+    point, start = _extract_coords(action)
+    if point is None and start is None:
+        return png_bytes
+    try:
+        img = Image.open(io.BytesIO(png_bytes)).convert("RGBA")
+        overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
+        draw = ImageDraw.Draw(overlay)
+        red = (255, 30, 30, 240)
+        radius = 5
+        if point and start:
+            draw.line([start, point], fill=red, width=2)
+            for px, py in (start, point):
+                draw.ellipse([px - radius, py - radius, px + radius, py + radius], fill=red)
+        else:
+            cx, cy = point or start  # type: ignore
+            draw.ellipse([cx - radius, cy - radius, cx + radius, cy + radius], fill=red)
+        out = Image.alpha_composite(img, overlay).convert("RGB")
+        buf = io.BytesIO()
+        out.save(buf, format="PNG")
+        return buf.getvalue()
+    except Exception as e:
+        _log.warning(f"annotate_click failed: {e}")
+        return png_bytes
+
+
+async def execute_action(
+    sandbox: ComputerSandbox,
+    action: dict[str, Any],
+    images_dir: Path,
+    step_idx: int,
+    key_aliases: Mapping[str, str] | None = None,
+) -> tuple[str, bytes | None, str | None]:
+    """Execute a computer-use action; returns (result_text, ss_bytes, ss_path)."""
+    command = DesktopCommand.from_mapping(action)
+    img_name = f"step_{step_idx:03d}.png"
+    t0 = time.monotonic()
+
+    if command.action is DesktopAction.SCREENSHOT:
+        ss = await _take_screenshot_with_recovery(sandbox, command.label)
+        if ss is None:
+            return "Screenshot failed after retries", None, None
+        (images_dir / img_name).write_bytes(ss)
+        _log.info(f"action screenshot: {time.monotonic() - t0:.2f}s, {len(ss)} bytes")
+        return "Screenshot taken", ss, f"images/{img_name}"
+
+    try:
+        if command.action in {DesktopAction.LEFT_CLICK, DesktopAction.CLICK}:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+            )
+        elif command.action is DesktopAction.RIGHT_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.RIGHT.value,
+            )
+        elif command.action is DesktopAction.DOUBLE_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+                double=True,
+            )
+        elif command.action is DesktopAction.TRIPLE_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+                double=True,
+            )
+            await asyncio.sleep(0.05)
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.LEFT.value,
+            )
+        elif command.action is DesktopAction.MIDDLE_CLICK:
+            await sandbox.mouse.click(
+                command.point.x,
+                command.point.y,
+                button=MouseButton.MIDDLE.value,
+            )
+        elif command.action is DesktopAction.TYPE:
+            await sandbox.keyboard.type(command.text)
+        elif command.action is DesktopAction.KEY:
+            key = normalize_key(command.key or command.text, key_aliases)
+            if "+" in key:
+                await sandbox.keyboard.hotkey(key)
+            else:
+                await sandbox.keyboard.press(key)
+        elif command.action is DesktopAction.SCROLL:
+            await sandbox.mouse.scroll(
+                command.point.x,
+                command.point.y,
+                command.direction.value,
+                command.amount,
+            )
+        elif command.action is DesktopAction.MOVE:
+            await sandbox.mouse.move(command.point.x, command.point.y)
+        elif command.action is DesktopAction.DRAG:
+            await sandbox.mouse.drag(
+                command.start_point.x,
+                command.start_point.y,
+                command.point.x,
+                command.point.y,
+            )
+        elif command.action is DesktopAction.WAIT:
+            await asyncio.sleep(command.duration)
+        else:
+            return f"Unknown action: {command.label}", None, None
+    except httpx.HTTPStatusError as e:
+        detail = e.response.text.strip().replace("\n", " ")[:240]
+        msg = f"Action '{command.label}' failed: HTTP {e.response.status_code} {detail}"
+        _log.warning(msg)
+        ss = await _take_screenshot_with_recovery(sandbox, command.label)
+        if ss is None:
+            return msg, None, None
+        (images_dir / img_name).write_bytes(ss)
+        return msg, ss, f"images/{img_name}"
+    except httpx.HTTPError as e:
+        msg = f"Action '{command.label}' failed: {type(e).__name__}: {e}"
+        _log.warning(msg)
+        ss = await _take_screenshot_with_recovery(sandbox, command.label)
+        if ss is None:
+            return msg, None, None
+        (images_dir / img_name).write_bytes(ss)
+        return msg, ss, f"images/{img_name}"
+
+    action_elapsed = time.monotonic() - t0
+    await asyncio.sleep(2)
+    ss = await _take_screenshot_with_recovery(sandbox, command.label)
+    if ss is None:
+        return (
+            f"Action '{command.label}' executed; screenshot failed after retries",
+            None,
+            None,
+        )
+    annotated = _annotate_click(ss, action)
+    (images_dir / img_name).write_bytes(annotated)
+    total_elapsed = time.monotonic() - t0
+    screenshot_elapsed = total_elapsed - action_elapsed
+    _log.info(
+        f"action {command.label}: {action_elapsed:.2f}s + "
+        f"screenshot {screenshot_elapsed:.2f}s = {total_elapsed:.2f}s total"
+    )
+    return f"Action '{command.label}' executed", ss, f"images/{img_name}"
+
+
+async def _take_screenshot_with_recovery(
+    sandbox: ComputerSandbox, action_type: str
+) -> bytes | None:
+    attempts = 4
+    for attempt in range(1, attempts + 1):
+        try:
+            return await sandbox.screenshot.take_full_screen()
+        except httpx.HTTPStatusError as e:
+            if attempt == attempts:
+                _log.warning(
+                    f"screenshot after {action_type} failed after retries: "
+                    f"HTTP {e.response.status_code} {e.response.text[:160]}"
+                )
+                return None
+            _log.warning(
+                f"screenshot after {action_type} failed: HTTP {e.response.status_code}; "
+                f"retrying in 10s ({attempt}/{attempts - 1})"
+            )
+            await asyncio.sleep(10)
+        except httpx.HTTPError as e:
+            if attempt == attempts:
+                _log.warning(
+                    f"screenshot after {action_type} failed after retries: {type(e).__name__}: {e}"
+                )
+                return None
+            _log.warning(
+                f"screenshot after {action_type} failed: {type(e).__name__}; "
+                f"retrying in 10s ({attempt}/{attempts - 1})"
+            )
+            await asyncio.sleep(10)