use-computer 0.0.14__tar.gz → 0.0.16__tar.gz

{use_computer-0.0.14 → use_computer-0.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: use-computer
-Version: 0.0.14
+Version: 0.0.16
 Summary: Python SDK for the use.computer macOS and iOS Computer Use API
 Project-URL: Homepage, https://use.computer
 Project-URL: Documentation, https://api.use.computer/docs
@@ -30,21 +30,29 @@ export USE_COMPUTER_API_KEY=mk_live_...
 ```
 
 ```python
-from use_computer import Computer
+from use_computer import Computer, SandboxType, SimulatorFamily
 
 with Computer().create() as mac:
     mac.exec_ssh("open -a TextEdit")
     mac.keyboard.type("hello")
     png = mac.screenshot.take_full_screen()
+
+with Computer().create(type=SandboxType.IOS, family=SimulatorFamily.TV) as tv:
+    tv.screenshot.take_full_screen()
+    tv.input.press_remote("select")
 ```
 
 Full DSL reference (macOS + simulator): [docs.use.computer/docs/sdk](https://docs.use.computer/docs/sdk)
 
-Simulator sandboxes use `type="ios"` for the SDK/API route, but `device_type`
-and `runtime` can target any installed compatible CoreSimulator pair: iPhone or
-iPad with iOS, Apple Watch with watchOS, or Apple TV with tvOS. Vision is
-unavailable for now. If omitted, the gateway defaults to iPhone 17 Pro on the
-latest installed iOS runtime.
+Simulator sandboxes use `type=SandboxType.IOS` for the SDK route, but
+`device_type` and `runtime` can target any installed compatible CoreSimulator
+pair: iPhone or iPad with iOS, Apple Watch with watchOS, Apple TV with tvOS, or
+Apple Vision with visionOS. Prefer `family=SimulatorFamily.TV`/`WATCH`/`VISION`
+unless you need to pin raw CoreSimulator identifiers. Raw strings like
+`type="ios"` still work for compatibility. If omitted, the gateway defaults to
+iPhone 17 Pro on the latest installed iOS runtime.
+Known-incompatible simulator types are filtered from family selection,
+including the non-4K Apple Vision Pro type on current fleet runtimes.
 
 ## Examples
 

{use_computer-0.0.14 → use_computer-0.0.16}/README.md

@@ -8,21 +8,29 @@ export USE_COMPUTER_API_KEY=mk_live_...
 ```
 
 ```python
-from use_computer import Computer
+from use_computer import Computer, SandboxType, SimulatorFamily
 
 with Computer().create() as mac:
     mac.exec_ssh("open -a TextEdit")
     mac.keyboard.type("hello")
     png = mac.screenshot.take_full_screen()
+
+with Computer().create(type=SandboxType.IOS, family=SimulatorFamily.TV) as tv:
+    tv.screenshot.take_full_screen()
+    tv.input.press_remote("select")
 ```
 
 Full DSL reference (macOS + simulator): [docs.use.computer/docs/sdk](https://docs.use.computer/docs/sdk)
 
-Simulator sandboxes use `type="ios"` for the SDK/API route, but `device_type`
-and `runtime` can target any installed compatible CoreSimulator pair: iPhone or
-iPad with iOS, Apple Watch with watchOS, or Apple TV with tvOS. Vision is
-unavailable for now. If omitted, the gateway defaults to iPhone 17 Pro on the
-latest installed iOS runtime.
+Simulator sandboxes use `type=SandboxType.IOS` for the SDK route, but
+`device_type` and `runtime` can target any installed compatible CoreSimulator
+pair: iPhone or iPad with iOS, Apple Watch with watchOS, Apple TV with tvOS, or
+Apple Vision with visionOS. Prefer `family=SimulatorFamily.TV`/`WATCH`/`VISION`
+unless you need to pin raw CoreSimulator identifiers. Raw strings like
+`type="ios"` still work for compatibility. If omitted, the gateway defaults to
+iPhone 17 Pro on the latest installed iOS runtime.
+Known-incompatible simulator types are filtered from family selection,
+including the non-4K Apple Vision Pro type on current fleet runtimes.
 
 ## Examples
 

{use_computer-0.0.14 → use_computer-0.0.16}/examples/_2_hello_ios.py

@@ -1,6 +1,6 @@
-from use_computer import Computer
+from use_computer import Computer, SandboxType
 
-with Computer().create(type="ios") as ios:
+with Computer().create(type=SandboxType.IOS) as ios:
     ios.apps.open_url("https://example.com")
     png = ios.screenshot.take_full_screen()
     open("hello_ios.png", "wb").write(png)

{use_computer-0.0.14 → use_computer-0.0.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer"
-version = "0.0.14"
+version = "0.0.16"
 description = "Python SDK for the use.computer macOS and iOS Computer Use API"
 readme = "README.md"
 requires-python = ">=3.10"

{use_computer-0.0.14 → use_computer-0.0.16}/use_computer/__init__.py

@@ -1,6 +1,8 @@
 from use_computer.client import AsyncComputer, Computer, RunStatus
 from use_computer.errors import PlatformNotSupportedError, UseComputerError
+from use_computer.ios.input import Button, Key, RemoteButton
 from use_computer.models import (
+    AccessibilityTree,
     ActionResult,
     ActResult,
     CursorPosition,
@@ -18,31 +20,51 @@ from use_computer.sandbox import (
     Sandbox,
     SandboxType,
 )
+from use_computer.simulators import (
+    SimulatorChoice,
+    SimulatorFamily,
+    family_for_device,
+    is_usable_device_type,
+    required_runtime_os,
+    runtime_os,
+    select_simulator,
+)
 from use_computer.tasks import Task, TasksClient, TaskSummary
 
 __all__ = [
     "Action",
+    "AccessibilityTree",
     "ActionResult",
     "ActResult",
     "AsyncIOSSandbox",
     "AsyncMacOSSandbox",
     "AsyncComputer",
     "AsyncSandbox",
+    "Button",
+    "Computer",
     "CursorPosition",
     "DisplayInfo",
     "ExecResult",
     "IOSSandbox",
+    "Key",
     "MacOSSandbox",
-    "Computer",
-    "UseComputerError",
     "PlatformNotSupportedError",
     "RecordingInfo",
+    "RemoteButton",
     "RunStatus",
     "Sandbox",
     "SandboxType",
+    "SimulatorChoice",
+    "SimulatorFamily",
     "Task",
     "TaskSummary",
     "TasksClient",
+    "UseComputerError",
+    "family_for_device",
+    "is_usable_device_type",
     "parse_pyautogui",
     "parse_xdotool",
+    "required_runtime_os",
+    "runtime_os",
+    "select_simulator",
 ]

use_computer-0.0.16/use_computer/accessibility.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import httpx
+
+from use_computer.models import AccessibilityTree
+
+
+class Accessibility:
+    def __init__(self, http: httpx.Client, prefix: str):
+        self._http = http
+        self._prefix = prefix
+
+    def get_tree(self, *, best_effort: bool = True) -> AccessibilityTree:
+        params = {"best_effort": "1"} if best_effort else None
+        resp = self._http.get(f"{self._prefix}/display/windows", params=params)
+        if best_effort and resp.status_code >= 400:
+            return AccessibilityTree(available=False, error=resp.text)
+        resp.raise_for_status()
+        data = resp.json()
+        if isinstance(data, dict) and "available" in data:
+            return AccessibilityTree.from_dict(data)
+        return AccessibilityTree(available=True, tree=data)
+
+
+class AsyncAccessibility:
+    def __init__(self, http: httpx.AsyncClient, prefix: str):
+        self._http = http
+        self._prefix = prefix
+
+    async def get_tree(self, *, best_effort: bool = True) -> AccessibilityTree:
+        params = {"best_effort": "1"} if best_effort else None
+        resp = await self._http.get(f"{self._prefix}/display/windows", params=params)
+        if best_effort and resp.status_code >= 400:
+            return AccessibilityTree(available=False, error=resp.text)
+        resp.raise_for_status()
+        data = resp.json()
+        if isinstance(data, dict) and "available" in data:
+            return AccessibilityTree.from_dict(data)
+        return AccessibilityTree(available=True, tree=data)

{use_computer-0.0.14 → use_computer-0.0.16}/use_computer/client.py

@@ -19,10 +19,13 @@ from use_computer.sandbox import (
     Sandbox,
     SandboxType,
 )
+from use_computer.simulators import SimulatorFamily, select_simulator
 from use_computer.tasks import TasksClient
 
 DEFAULT_BASE_URL = "https://api.use.computer"
 
+SandboxTypeInput = SandboxType | Literal["macos", "ios"]
+
 
 def _resolve_api_key(api_key: str | None) -> str | None:
     if api_key is not None:
@@ -34,6 +37,12 @@ def _resolve_base_url(base_url: str | None) -> str:
     return (base_url or os.getenv("USE_COMPUTER_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
 
 
+def _normalize_sandbox_type(type: SandboxTypeInput) -> SandboxType:
+    if isinstance(type, SandboxType):
+        return type
+    return SandboxType(type)
+
+
 @dataclass
 class RunStatus:
     """Status of an ad-hoc or model run."""
@@ -102,32 +111,50 @@ class Computer:
         return resp.json()
 
     @overload
-    def create(self, *, type: Literal["macos"] = ..., host: str = ...) -> MacOSSandbox: ...
+    def create(
+        self,
+        *,
+        type: Literal["macos", SandboxType.MACOS] = ...,
+        host: str = ...,
+    ) -> MacOSSandbox: ...
+
     @overload
     def create(
-        self, *, type: Literal["ios"], device_type: str = ..., runtime: str = ...
+        self,
+        *,
+        type: Literal["ios", SandboxType.IOS],
+        family: SimulatorFamily | str = ...,
+        device_type: str = ...,
+        runtime: str = ...,
     ) -> IOSSandbox: ...
 
     def create(
         self,
         *,
-        type: str = "macos",
+        type: SandboxTypeInput = SandboxType.MACOS,
         host: str = "",
+        family: SimulatorFamily | str | None = None,
         device_type: str = "",
         runtime: str = "",
     ) -> MacOSSandbox | IOSSandbox:
         """Create a new sandbox.
 
         Args:
-            type: "macos" (default) or "ios".
+            type: SandboxType.MACOS (default), SandboxType.IOS, or the raw API string.
             host: Pin sandbox to a specific host machine (e.g. "mm001").
-            device_type: iOS only — simulator device type identifier.
-            runtime: iOS only — simulator runtime identifier.
+            family: Simulator family enum/alias, for example SimulatorFamily.TV.
+            device_type: Simulator device type identifier, for exact pinning.
+            runtime: Simulator runtime identifier, for exact pinning.
         """
-        body: dict = {"type": type}
+        sandbox_type = _normalize_sandbox_type(type)
+        body: dict = {"type": sandbox_type.value}
         if host:
             body["host"] = host
-        if type == "ios":
+        if sandbox_type == SandboxType.IOS:
+            if family and (not device_type or not runtime):
+                choice = select_simulator(self.platforms(), family)
+                device_type = device_type or choice.device_type
+                runtime = runtime or choice.runtime
             if device_type:
                 body["device_type"] = device_type
             if runtime:
@@ -138,7 +165,7 @@ class Computer:
         data = resp.json()
         sid = data["sandbox_id"]
 
-        if type == "ios":
+        if sandbox_type == SandboxType.IOS:
             return IOSSandbox(sandbox_id=sid, http=self._http)
         return MacOSSandbox(
             sandbox_id=sid,
@@ -249,33 +276,49 @@ class AsyncComputer:
 
     @overload
     async def create(
-        self, *, type: Literal["macos"] = ..., host: str = ...
+        self,
+        *,
+        type: Literal["macos", SandboxType.MACOS] = ...,
+        host: str = ...,
     ) -> AsyncMacOSSandbox: ...
+
     @overload
     async def create(
-        self, *, type: Literal["ios"], device_type: str = ..., runtime: str = ...
+        self,
+        *,
+        type: Literal["ios", SandboxType.IOS],
+        family: SimulatorFamily | str = ...,
+        device_type: str = ...,
+        runtime: str = ...,
     ) -> AsyncIOSSandbox: ...
 
     async def create(
         self,
         *,
-        type: str = "macos",
+        type: SandboxTypeInput = SandboxType.MACOS,
         host: str = "",
+        family: SimulatorFamily | str | None = None,
         device_type: str = "",
         runtime: str = "",
     ) -> AsyncMacOSSandbox | AsyncIOSSandbox:
         """Create a new sandbox.
 
         Args:
-            type: "macos" (default) or "ios".
+            type: SandboxType.MACOS (default), SandboxType.IOS, or the raw API string.
             host: Pin sandbox to a specific host machine (e.g. "mm001").
-            device_type: iOS only — simulator device type identifier.
-            runtime: iOS only — simulator runtime identifier.
+            family: Simulator family enum/alias, for example SimulatorFamily.TV.
+            device_type: Simulator device type identifier, for exact pinning.
+            runtime: Simulator runtime identifier, for exact pinning.
         """
-        body: dict = {"type": type}
+        sandbox_type = _normalize_sandbox_type(type)
+        body: dict = {"type": sandbox_type.value}
         if host:
             body["host"] = host
-        if type == "ios":
+        if sandbox_type == SandboxType.IOS:
+            if family and (not device_type or not runtime):
+                choice = select_simulator(await self.platforms(), family)
+                device_type = device_type or choice.device_type
+                runtime = runtime or choice.runtime
             if device_type:
                 body["device_type"] = device_type
             if runtime:
@@ -286,7 +329,7 @@ class AsyncComputer:
         data = resp.json()
         sid = data["sandbox_id"]
 
-        if type == "ios":
+        if sandbox_type == SandboxType.IOS:
             return AsyncIOSSandbox(sandbox_id=sid, http=self._http)
         return AsyncMacOSSandbox(
             sandbox_id=sid,

{use_computer-0.0.14 → use_computer-0.0.16}/use_computer/ios/input.py

@@ -17,6 +17,19 @@ class Button(str, Enum):
     APPLE_PAY = "apple-pay"
 
 
+class RemoteButton(str, Enum):
+    """Apple TV remote-style controls."""
+
+    UP = "up"
+    DOWN = "down"
+    LEFT = "left"
+    RIGHT = "right"
+    SELECT = "select"
+    MENU = "menu"
+    HOME = "home"
+    PLAY_PAUSE = "play-pause"
+
+
 class Key(int, Enum):
     """Common HID keycodes for iOS simulator key presses."""
 
@@ -79,6 +92,13 @@ class Input:
         resp.raise_for_status()
         return ActionResult.from_dict(resp.json())
 
+    def press_remote(self, button: RemoteButton | str) -> ActionResult:
+        """Press an Apple TV remote-style control."""
+        val = button.value if isinstance(button, RemoteButton) else button
+        resp = self._http.post(f"{self._prefix}/remote", json={"button": val})
+        resp.raise_for_status()
+        return ActionResult.from_dict(resp.json())
+
 
 class AsyncInput:
     def __init__(self, http: httpx.AsyncClient, prefix: str):
@@ -116,3 +136,10 @@ class AsyncInput:
         resp = await self._http.post(f"{self._prefix}/key", json={"keycode": val})
         resp.raise_for_status()
         return ActionResult.from_dict(resp.json())
+
+    async def press_remote(self, button: RemoteButton | str) -> ActionResult:
+        """Press an Apple TV remote-style control."""
+        val = button.value if isinstance(button, RemoteButton) else button
+        resp = await self._http.post(f"{self._prefix}/remote", json={"button": val})
+        resp.raise_for_status()
+        return ActionResult.from_dict(resp.json())

{use_computer-0.0.14 → use_computer-0.0.16}/use_computer/models.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from typing import Any
 
 
 @dataclass
@@ -29,6 +30,9 @@ class DisplayInfo:
     width: int
     height: int
     scale: float = 1.0
+    platform: str = ""
+    device_type: str = ""
+    runtime: str = ""
 
     @classmethod
     def from_dict(cls, d: dict) -> DisplayInfo:
@@ -36,6 +40,28 @@ class DisplayInfo:
             width=d.get("width", 0),
             height=d.get("height", 0),
             scale=d.get("scale", 1.0),
+            platform=d.get("platform", ""),
+            device_type=(
+                d.get("device_type") or d.get("deviceType") or d.get("deviceTypeIdentifier", "")
+            ),
+            runtime=d.get("runtime", ""),
+        )
+
+
+@dataclass
+class AccessibilityTree:
+    """Best-effort simulator accessibility tree result."""
+
+    available: bool
+    tree: Any = None
+    error: str = ""
+
+    @classmethod
+    def from_dict(cls, d: dict) -> AccessibilityTree:
+        return cls(
+            available=bool(d.get("available", False)),
+            tree=d.get("tree"),
+            error=d.get("error", ""),
         )
 
 

{use_computer-0.0.14 → use_computer-0.0.16}/use_computer/sandbox.py

@@ -9,6 +9,7 @@ from pathlib import Path
 
 import httpx
 
+from use_computer.accessibility import Accessibility, AsyncAccessibility
 from use_computer.display import AsyncDisplay, Display
 from use_computer.ios.apps import Apps, AsyncApps
 from use_computer.ios.environment import AsyncEnvironment, Environment
@@ -104,6 +105,7 @@ class Sandbox:
         self.screenshot = Screenshot(http, self._prefix)
         self.recording = Recording(http, self._prefix)
         self.display = Display(http, self._prefix)
+        self.accessibility = Accessibility(http, self._prefix)
 
     def upload(self, local_path: str | Path, remote_path: str) -> None:
         with open(local_path, "rb") as f:
@@ -279,6 +281,7 @@ class AsyncSandbox:
         self.screenshot = AsyncScreenshot(http, self._prefix)
         self.recording = AsyncRecording(http, self._prefix)
         self.display = AsyncDisplay(http, self._prefix)
+        self.accessibility = AsyncAccessibility(http, self._prefix)
 
     async def upload(self, local_path: str | Path, remote_path: str) -> None:
         with open(local_path, "rb") as f:

use_computer-0.0.16/use_computer/simulators.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class SimulatorFamily(str, Enum):
+    """CoreSimulator device families supported by the API."""
+
+    IPHONE = "iPhone"
+    IPAD = "iPad"
+    WATCH = "Apple Watch"
+    TV = "Apple TV"
+    VISION = "Apple Vision"
+
+
+@dataclass(frozen=True)
+class SimulatorChoice:
+    """Resolved CoreSimulator device/runtime pair."""
+
+    family: SimulatorFamily
+    device_type: str
+    runtime: str
+    device_name: str = ""
+    runtime_name: str = ""
+
+
+_FAMILY_RUNTIME_OS: dict[SimulatorFamily, str] = {
+    SimulatorFamily.IPHONE: "iOS",
+    SimulatorFamily.IPAD: "iOS",
+    SimulatorFamily.WATCH: "watchOS",
+    SimulatorFamily.TV: "tvOS",
+    SimulatorFamily.VISION: "visionOS",
+}
+
+_RUNTIME_OS_ALIASES = {
+    "xrOS": "visionOS",
+}
+
+
+def normalize_simulator_family(family: SimulatorFamily | str) -> SimulatorFamily:
+    """Normalize SDK enum values and common user-facing aliases."""
+
+    if isinstance(family, SimulatorFamily):
+        return family
+    compact = re.sub(r"[^a-z0-9]", "", family.lower())
+    aliases = {
+        "iphone": SimulatorFamily.IPHONE,
+        "ios": SimulatorFamily.IPHONE,
+        "ipad": SimulatorFamily.IPAD,
+        "watch": SimulatorFamily.WATCH,
+        "applewatch": SimulatorFamily.WATCH,
+        "watchos": SimulatorFamily.WATCH,
+        "tv": SimulatorFamily.TV,
+        "appletv": SimulatorFamily.TV,
+        "tvos": SimulatorFamily.TV,
+        "vision": SimulatorFamily.VISION,
+        "applevision": SimulatorFamily.VISION,
+        "visionpro": SimulatorFamily.VISION,
+        "applevisionpro": SimulatorFamily.VISION,
+        "visionos": SimulatorFamily.VISION,
+        "xros": SimulatorFamily.VISION,
+    }
+    try:
+        return aliases[compact]
+    except KeyError as exc:
+        raise ValueError(f"unknown simulator family: {family!r}") from exc
+
+
+def family_for_device(device: dict[str, Any] | str) -> SimulatorFamily | None:
+    """Return the simulator family for a device type dict, identifier, or name."""
+
+    text = (
+        device
+        if isinstance(device, str)
+        else f"{device.get('name', '')} {device.get('identifier', '')}"
+    )
+    if re.search(r"Apple[- ]Watch", text, re.I):
+        return SimulatorFamily.WATCH
+    if re.search(r"Apple[- ]TV", text, re.I):
+        return SimulatorFamily.TV
+    if re.search(r"Apple[- ]Vision", text, re.I):
+        return SimulatorFamily.VISION
+    if re.search(r"iPad", text, re.I):
+        return SimulatorFamily.IPAD
+    if re.search(r"iPhone", text, re.I):
+        return SimulatorFamily.IPHONE
+    return None
+
+
+def runtime_os(runtime: dict[str, Any] | str) -> str:
+    """Return the normalized OS family for a runtime dict or identifier."""
+
+    identifier = runtime if isinstance(runtime, str) else runtime.get("identifier", "")
+    match = re.search(r"SimRuntime\.([^-]+)-", identifier)
+    if not match:
+        return ""
+    os_name = match.group(1)
+    return _RUNTIME_OS_ALIASES.get(os_name, os_name)
+
+
+def required_runtime_os(family: SimulatorFamily | str) -> str:
+    """Return the runtime OS required by a simulator family."""
+
+    return _FAMILY_RUNTIME_OS[normalize_simulator_family(family)]
+
+
+def select_simulator(
+    platforms: dict[str, Any],
+    family: SimulatorFamily | str,
+    *,
+    device_name: str = "",
+) -> SimulatorChoice:
+    """Select a compatible device/runtime pair from `/v1/platforms` output."""
+
+    fam = normalize_simulator_family(family)
+    ios = platforms.get("ios") or {}
+    devices = [
+        d
+        for d in ios.get("device_types") or []
+        if family_for_device(d) == fam and is_usable_device_type(d)
+    ]
+    if device_name:
+        needle = device_name.lower()
+        devices = [d for d in devices if needle in (d.get("name") or "").lower()]
+    if not devices:
+        raise ValueError(f"no {fam.value} simulator devices available")
+
+    want_os = required_runtime_os(fam)
+    runtimes = [
+        rt
+        for rt in ios.get("runtimes") or []
+        if rt.get("isAvailable", True) and runtime_os(rt) == want_os
+    ]
+    if not runtimes:
+        raise ValueError(f"no {want_os} runtime available for {fam.value}")
+
+    devices.sort(key=lambda d: d.get("name") or d.get("identifier") or "")
+    runtimes.sort(
+        key=lambda rt: (rt.get("version") or "", rt.get("identifier") or ""),
+        reverse=True,
+    )
+    device = _default_device(fam, devices)
+    runtime = runtimes[0]
+    return SimulatorChoice(
+        family=fam,
+        device_type=device.get("identifier", ""),
+        runtime=runtime.get("identifier", ""),
+        device_name=device.get("name", ""),
+        runtime_name=runtime.get("name", ""),
+    )
+
+
+def _default_device(family: SimulatorFamily, devices: list[dict[str, Any]]) -> dict[str, Any]:
+    if family == SimulatorFamily.IPHONE:
+        for d in devices:
+            name = d.get("name") or ""
+            if re.search(r"17 Pro\b", name) and "Max" not in name:
+                return d
+    if family == SimulatorFamily.VISION:
+        for d in devices:
+            if "Apple-Vision-Pro-4K" in (d.get("identifier") or ""):
+                return d
+    return devices[0]
+
+
+def is_usable_device_type(device: dict[str, Any] | str) -> bool:
+    """Return false for simulator device types known to fail with current runtimes."""
+
+    text = (
+        device
+        if isinstance(device, str)
+        else f"{device.get('name', '')} {device.get('identifier', '')}"
+    )
+    if re.search(r"Apple[- ]Vision[- ]Pro", text, re.I):
+        return "Apple-Vision-Pro-4K" in text or re.search(r"\b4K\b", text, re.I) is not None
+    return True