use-computer 0.0.38__tar.gz → 0.0.40__tar.gz

{use_computer-0.0.38 → use_computer-0.0.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: use-computer
-Version: 0.0.38
+Version: 0.0.40
 Summary: Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu
 Project-URL: Homepage, https://use.computer
 Project-URL: Documentation, https://api.use.computer/docs
@@ -24,6 +24,8 @@ Requires-Dist: google-genai>=1; extra == 'agents'
 Requires-Dist: litellm>=1; extra == 'agents'
 Requires-Dist: openai>=1; extra == 'agents'
 Requires-Dist: pillow>=10; extra == 'agents'
+Requires-Dist: tinker-cookbook>=0.1.0; (python_full_version >= '3.11') and extra == 'agents'
+Requires-Dist: tinker>=0.14.0; (python_full_version >= '3.11') and extra == 'agents'
 Provides-Extra: harbor
 Requires-Dist: harbor>=0.4.0; (python_version >= '3.12') and extra == 'harbor'
 Requires-Dist: pyyaml>=6; extra == 'harbor'

{use_computer-0.0.38 → use_computer-0.0.40}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer"
-version = "0.0.38"
+version = "0.0.40"
 description = "Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -30,6 +30,8 @@ agents = [
     "anthropic>=0.86",
     "litellm>=1",
     "google-genai>=1",
+    "tinker>=0.14.0 ; python_full_version >= '3.11'",
+    "tinker-cookbook>=0.1.0 ; python_full_version >= '3.11'",
 ]
 harbor = [
     "harbor>=0.4.0; python_version >= '3.12'",

{use_computer-0.0.38 → use_computer-0.0.40}/use_computer/agents/base/recordings.py

@@ -11,6 +11,11 @@ from use_computer.sandbox import AsyncSandbox
 
 class RecordingMixin:
     async def start_recording(self: Any, sandbox: AsyncSandbox) -> None:
+        if getattr(getattr(sandbox, "type", None), "value", "") == "ios":
+            try:
+                await sandbox.screenshot.take_full_screen()
+            except Exception as e:
+                self.logger.warning(f"iOS recording warm-up screenshot failed: {e}")
         try:
             result = await sandbox.recording.start(name="trial")
             self._recording_id = result.id
@@ -22,6 +27,7 @@ class RecordingMixin:
         if not self._recording_id:
             return
         await asyncio.sleep(3.0)
+        rec_path = self.logs_dir / "recording.mp4"
         try:
             info = await sandbox.recording.stop(self._recording_id)
             self.logger.info(
@@ -33,16 +39,24 @@ class RecordingMixin:
 
             await asyncio.sleep(2)
 
-            rec_path = self.logs_dir / "recording.mp4"
-            self.logger.info(f"Downloading recording → {rec_path}")
-            t_dl = time.monotonic()
-            await sandbox.recording.download(self._recording_id, str(rec_path), timeout=180.0)
-            dl_elapsed = time.monotonic() - t_dl
-            local_size = rec_path.stat().st_size
-            self.logger.info(
-                f"Recording saved: {rec_path} "
-                f"local_size={local_size}B ({local_size / (1024 * 1024):.2f} MB) "
-                f"download_elapsed={dl_elapsed:.1f}s"
-            )
+            await self._download_recording(sandbox, rec_path)
         except Exception as e:
             self.logger.warning(f"Recording save failed: {type(e).__name__}: {e}")
+            try:
+                await self._download_recording(sandbox, rec_path)
+            except Exception as fallback:
+                self.logger.warning(
+                    f"Recording fallback download failed: {type(fallback).__name__}: {fallback}"
+                )
+
+    async def _download_recording(self: Any, sandbox: AsyncSandbox, rec_path) -> None:
+        self.logger.info(f"Downloading recording → {rec_path}")
+        t_dl = time.monotonic()
+        await sandbox.recording.download(self._recording_id, str(rec_path), timeout=180.0)
+        dl_elapsed = time.monotonic() - t_dl
+        local_size = rec_path.stat().st_size
+        self.logger.info(
+            f"Recording saved: {rec_path} "
+            f"local_size={local_size}B ({local_size / (1024 * 1024):.2f} MB) "
+            f"download_elapsed={dl_elapsed:.1f}s"
+        )

{use_computer-0.0.38 → use_computer-0.0.40}/use_computer/agents/providers/generic.py

@@ -83,6 +83,11 @@ class GenericCUAAgent(BaseCUAAgent):
         coord_mode: str | None = None,
         reasoning_effort: str | None = None,
         reasoning_history: str | None = None,
+        renderer_name: str | None = None,
+        model_path: str | None = None,
+        temperature: float = 1.0,
+        top_p: float = 0.95,
+        top_k: int = -1,
         **kwargs: Any,
     ):
         super().__init__(logs_dir=logs_dir, model_name=model_name, **kwargs)
@@ -106,7 +111,12 @@ class GenericCUAAgent(BaseCUAAgent):
                 ReasoningHistory(reasoning_history) if reasoning_history is not None else None
             ),
         ).as_dict()
-        self._tinker_llm = None
+        self._tinker_backend = None
+        self._renderer_name = renderer_name
+        self._model_path = model_path
+        self._temperature = temperature
+        self._top_p = top_p
+        self._top_k = top_k
         # Coordinate convention emitted by the model:
         #   "api_pixel"        — Kimi default: 0..1 floats OR pixel ints in API space (1280x800)
         #   "normalized_1000"  — Qwen-VL default: 0..1000 ints as per-mille of screen
@@ -168,39 +178,19 @@ class GenericCUAAgent(BaseCUAAgent):
     async def _tinker_completion(
         self, model: str, messages: list[dict], max_tokens: int
     ) -> tuple[str, int, int]:
-        if self._tinker_llm is None:
-            from harbor.llms.tinker import TinkerLLM
+        if self._tinker_backend is None:
+            from use_computer.agents.providers.tinker_backend import TinkerBackend
 
-            self._tinker_llm = TinkerLLM(
+            self._tinker_backend = TinkerBackend(
                 model_name=model,
+                model_path=self._model_path,
+                renderer_name=self._renderer_name,
+                temperature=self._temperature,
+                top_p=self._top_p,
+                top_k=self._top_k,
                 max_tokens=max_tokens,
             )
-
-        # TinkerLLM.call() expects (prompt, message_history)
-        # Split system + history from the final user message
-        history = messages[:-1] if len(messages) > 1 else []
-        prompt = ""
-        last_msg = messages[-1] if messages else {"content": ""}
-        content = last_msg.get("content")
-        if isinstance(content, str):
-            prompt = content
-        elif isinstance(content, list):
-            prompt = " ".join(
-                p.get("text", "")
-                for p in content
-                if isinstance(p, dict) and p.get("type") == "text"
-            )
-
-        response = await self._tinker_llm.call(prompt=prompt, message_history=history)
-        content = response.content or ""
-        text = (
-            content
-            if isinstance(content, str)
-            else "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in content)
-        )
-        prompt_tokens = response.usage.prompt_tokens if response.usage else 0
-        completion_tokens = response.usage.completion_tokens if response.usage else 0
-        return text, prompt_tokens, completion_tokens
+        return await self._tinker_backend.complete(messages, max_tokens)
 
     async def _run_agent(
         self,

use_computer-0.0.40/use_computer/agents/providers/tinker_backend.py

@@ -0,0 +1,163 @@
+"""Image-aware Tinker sampling backend for the generic CUA agent.
+
+harbor's TinkerLLM forwards text only, so a vision agent loses its screenshots
+on the tinker path. This backend builds the generation prompt through a
+tinker-cookbook renderer with the model's image processor attached, so the
+OpenAI-style `image_url` parts the generic agent emits reach the model.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import importlib
+import io
+from typing import Any, cast
+
+from PIL import Image
+
+
+class TinkerBackend:
+    """Vision-capable tinker sampler that consumes OpenAI chat messages."""
+
+    def __init__(
+        self,
+        *,
+        model_name: str,
+        model_path: str | None = None,
+        renderer_name: str | None = None,
+        temperature: float = 1.0,
+        top_p: float = 0.95,
+        top_k: int = -1,
+        max_tokens: int = 2048,
+        retry_times: int = 5,
+        retry_interval: float = 5.0,
+    ) -> None:
+        self.model_name = model_name
+        self.model_path = model_path
+        self.renderer_name = renderer_name
+        self.temperature = temperature
+        self.top_p = top_p
+        self.top_k = top_k
+        self.max_tokens = max_tokens
+        self.retry_times = retry_times
+        self.retry_interval = retry_interval
+        self._sampling_client = None
+        self._renderer = None
+
+    async def _ensure_ready(self) -> None:
+        if self._sampling_client is not None and self._renderer is not None:
+            return
+        try:
+            tinker = importlib.import_module("tinker")
+            get_image_processor = importlib.import_module(
+                "tinker_cookbook.image_processing_utils"
+            ).get_image_processor
+            get_model_attributes = importlib.import_module(
+                "tinker_cookbook.model_info"
+            ).get_model_attributes
+            get_renderer = importlib.import_module("tinker_cookbook.renderers").get_renderer
+            get_tokenizer = importlib.import_module("tinker_cookbook.tokenizer_utils").get_tokenizer
+        except ImportError as exc:
+            raise ImportError(
+                "TinkerBackend requires `tinker` and `tinker-cookbook`. "
+                "Install them with `uv pip install harbor[tinker]`."
+            ) from exc
+
+        service_client = tinker.ServiceClient()
+        if self.model_path:
+            self._sampling_client = await service_client.create_sampling_client_async(
+                model_path=self.model_path
+            )
+        else:
+            self._sampling_client = await service_client.create_sampling_client_async(
+                base_model=self.model_name
+            )
+
+        tokenizer = get_tokenizer(self.model_name)
+        attrs = get_model_attributes(self.model_name)
+        image_processor = get_image_processor(self.model_name) if attrs.is_vl else None
+        renderer_name = self.renderer_name or _default_renderer(self.model_name)
+        self._renderer = get_renderer(
+            renderer_name,
+            tokenizer,
+            image_processor=image_processor,
+            model_name=self.model_name,
+        )
+
+    async def complete(
+        self, messages: list[dict[str, Any]], max_tokens: int
+    ) -> tuple[str, int, int]:
+        """Sample one response; returns (text, prompt_tokens, completion_tokens)."""
+        await self._ensure_ready()
+        assert self._sampling_client is not None
+        assert self._renderer is not None
+
+        tinker = importlib.import_module("tinker")
+        get_text_content = importlib.import_module("tinker_cookbook.renderers").get_text_content
+
+        model_input = self._renderer.build_generation_prompt(
+            cast(Any, _to_tinker_messages(messages))
+        )
+        stop = self._renderer.get_stop_sequences()
+
+        last_error: Exception | None = None
+        for attempt in range(self.retry_times):
+            try:
+                response = await self._sampling_client.sample_async(
+                    prompt=model_input,
+                    num_samples=1,
+                    sampling_params=tinker.SamplingParams(
+                        temperature=self.temperature,
+                        top_p=self.top_p,
+                        top_k=self.top_k,
+                        max_tokens=max_tokens,
+                        stop=stop,
+                    ),
+                )
+                sequence = response.sequences[0]
+                parsed_message, _termination = self._renderer.parse_response(sequence.tokens)
+                return get_text_content(parsed_message), model_input.length, len(sequence.tokens)
+            except Exception as exc:
+                last_error = exc
+                if attempt == self.retry_times - 1:
+                    break
+                await asyncio.sleep(self.retry_interval)
+        assert last_error is not None
+        raise last_error
+
+
+def _to_tinker_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Convert OpenAI chat messages to tinker-cookbook messages (text + PIL images)."""
+    converted: list[dict[str, Any]] = []
+    for message in messages:
+        content = message.get("content")
+        if isinstance(content, str):
+            converted.append({"role": message["role"], "content": content})
+            continue
+
+        parts: list[dict[str, Any]] = []
+        for part in content or []:
+            if part.get("type") == "text":
+                parts.append({"type": "text", "text": part.get("text", "")})
+            elif part.get("type") == "image_url":
+                url = part.get("image_url", {}).get("url", "")
+                b64 = url.split(",", 1)[1] if "," in url else url
+                image = Image.open(io.BytesIO(base64.b64decode(b64))).convert("RGB")
+                parts.append({"type": "image", "image": image})
+        if len(parts) == 1 and parts[0]["type"] == "text":
+            converted.append({"role": message["role"], "content": parts[0]["text"]})
+        else:
+            converted.append({"role": message["role"], "content": parts})
+    return converted
+
+
+def _default_renderer(model_name: str) -> str:
+    get_recommended_renderer_names = importlib.import_module(
+        "tinker_cookbook.model_info"
+    ).get_recommended_renderer_names
+
+    recommended = get_recommended_renderer_names(model_name)
+    if "qwen3_5_disable_thinking" in recommended:
+        return "qwen3_5_disable_thinking"
+    return recommended[0]

{use_computer-0.0.38 → use_computer-0.0.40}/use_computer/client.py

@@ -53,6 +53,12 @@ def _normalize_sandbox_type(type: SandboxTypeInput) -> SandboxType:
     return SandboxType(type)
 
 
+def _expand_core_simulator_id(value: str, kind: str) -> str:
+    if not value or value.startswith("com.apple."):
+        return value
+    return f"com.apple.CoreSimulator.{kind}.{value}"
+
+
 @dataclass
 class RunStatus:
     """Status of an ad-hoc or model run."""
@@ -183,7 +189,7 @@ class Computer:
 
         Args:
             type: SandboxType.MACOS (default), IOS, WINDOWS, UBUNTU, or the raw API string.
-            host: Pin sandbox to a specific host machine (e.g. "mm001").
+            host: Pin sandbox to a specific host machine; for iOS this must be a Mini IP.
             family: Simulator family enum/alias, for example SimulatorFamily.TV.
             device_type: Simulator device type identifier, for exact pinning.
             runtime: Simulator runtime identifier, for exact pinning.
@@ -195,7 +201,7 @@ class Computer:
         """
         sandbox_type = _normalize_sandbox_type(type)
         body: dict = {"type": sandbox_type.value}
-        if host:
+        if host and sandbox_type != SandboxType.IOS:
             body["host"] = host
         if sandbox_type in (SandboxType.WINDOWS, SandboxType.UBUNTU):
             if version and snapshot:
@@ -207,14 +213,16 @@ class Computer:
             if resources:
                 body["resources"] = resources
         if sandbox_type == SandboxType.IOS:
+            if host:
+                body["mini_ip"] = host
             if family and (not device_type or not runtime):
                 choice = select_simulator(self.platforms(), family)
                 device_type = device_type or choice.device_type
                 runtime = runtime or choice.runtime
             if device_type:
-                body["device_type"] = device_type
+                body["device_type"] = _expand_core_simulator_id(device_type, "SimDeviceType")
             if runtime:
-                body["runtime"] = runtime
+                body["runtime"] = _expand_core_simulator_id(runtime, "SimRuntime")
 
         resp = self._http.post("/v1/sandboxes", json=body, timeout=180.0)
         resp.raise_for_status()
@@ -425,7 +433,7 @@ class AsyncComputer:
 
         Args:
             type: SandboxType.MACOS (default), IOS, WINDOWS, UBUNTU, or the raw API string.
-            host: Pin sandbox to a specific host machine (e.g. "mm001").
+            host: Pin sandbox to a specific host machine; for iOS this must be a Mini IP.
             family: Simulator family enum/alias, for example SimulatorFamily.TV.
             device_type: Simulator device type identifier, for exact pinning.
             runtime: Simulator runtime identifier, for exact pinning.
@@ -437,7 +445,7 @@ class AsyncComputer:
         """
         sandbox_type = _normalize_sandbox_type(type)
         body: dict = {"type": sandbox_type.value}
-        if host:
+        if host and sandbox_type != SandboxType.IOS:
             body["host"] = host
         if sandbox_type in (SandboxType.WINDOWS, SandboxType.UBUNTU):
             if version and snapshot:
@@ -449,14 +457,16 @@ class AsyncComputer:
             if resources:
                 body["resources"] = resources
         if sandbox_type == SandboxType.IOS:
+            if host:
+                body["mini_ip"] = host
             if family and (not device_type or not runtime):
                 choice = select_simulator(await self.platforms(), family)
                 device_type = device_type or choice.device_type
                 runtime = runtime or choice.runtime
             if device_type:
-                body["device_type"] = device_type
+                body["device_type"] = _expand_core_simulator_id(device_type, "SimDeviceType")
             if runtime:
-                body["runtime"] = runtime
+                body["runtime"] = _expand_core_simulator_id(runtime, "SimRuntime")
 
         resp = await self._http.post("/v1/sandboxes", json=body, timeout=180.0)
         resp.raise_for_status()

{use_computer-0.0.38 → use_computer-0.0.40}/use_computer/harbor/environment/platforms/__init__.py

@@ -104,12 +104,13 @@ class IOSAdapter(PlatformAdapter):
             env.logger.info(f"ios pin: device_type={device_type!r} runtime={runtime!r}")
         return await env._client.create(
             type="ios",
+            host=env._host,
             device_type=device_type,
             runtime=runtime,
         )
 
     async def setup(self, env: Any) -> None:
-        await asyncio.sleep(60.0)
+        await asyncio.sleep(2.0)
 
     async def exec(
         self,

{use_computer-0.0.38 → use_computer-0.0.40}/use_computer/harbor/environment/platforms/ubuntu.py

@@ -109,6 +109,8 @@ async def exec_local(
 
 
 async def run_setup(env: Any) -> None:
+    await wake_desktop(env)
+
     setup_dir = env._task_dir / "tests" / "setup"
 
     for setup_file in read_setup_files(setup_dir):
@@ -139,6 +141,28 @@ async def run_setup(env: Any) -> None:
     env.logger.info("task setup complete")
 
 
+async def wake_desktop(env: Any) -> None:
+    script = """
+echo 'user:lume' | sudo chpasswd || true
+export DISPLAY=:0
+export XAUTHORITY=/home/user/.Xauthority
+gsettings set org.gnome.desktop.screensaver lock-enabled false || true
+gsettings set org.gnome.desktop.session idle-delay 0 || true
+xset s off -dpms || true
+xset dpms force on || true
+xdotool key Escape || true
+sleep 1
+xdotool key ctrl+a || true
+xdotool type lume || true
+xdotool key Return || true
+sleep 3
+xset dpms force on || true
+"""
+    result = await env.ubuntu_sandbox.shell.run(script, shell="bash", timeout=60)
+    if result.return_code != 0:
+        env.logger.warning(f"ubuntu desktop wake rc={result.return_code}: {result.stderr or ''}")
+
+
 async def upload_file(env: Any, source: Path | str, target: str) -> None:
     await env.ubuntu_sandbox.upload(str(source), target)
 

{use_computer-0.0.38 → use_computer-0.0.40}/use_computer/sandbox.py

@@ -236,7 +236,11 @@ class MacOSSandbox(Sandbox):
         return ActResult(data=resp.json())
 
     def exec_ssh(self, command: str, timeout: int = 120) -> ExecResult:
-        resp = self._http.post(f"{self._prefix}/exec", json={"command": command}, timeout=timeout)
+        resp = self._http.post(
+            f"{self._prefix}/exec",
+            json={"command": command, "timeout": timeout},
+            timeout=timeout,
+        )
         resp.raise_for_status()
         return ExecResult.from_dict(resp.json())
 
@@ -254,7 +258,9 @@ class MacOSSandbox(Sandbox):
         command if you need brew binaries.
         """
         resp = self._http.post(
-            f"{self._prefix}/exec_ax", json={"command": command}, timeout=timeout
+            f"{self._prefix}/exec_ax",
+            json={"command": command, "timeout": timeout},
+            timeout=timeout,
         )
         resp.raise_for_status()
         return ExecResult.from_dict(resp.json())
@@ -650,7 +656,9 @@ class AsyncMacOSSandbox(AsyncSandbox):
 
     async def exec_ssh(self, command: str, timeout: int = 120) -> ExecResult:
         resp = await self._http.post(
-            f"{self._prefix}/exec", json={"command": command}, timeout=timeout
+            f"{self._prefix}/exec",
+            json={"command": command, "timeout": timeout},
+            timeout=timeout,
         )
         resp.raise_for_status()
         return ExecResult.from_dict(resp.json())
@@ -658,7 +666,9 @@ class AsyncMacOSSandbox(AsyncSandbox):
     async def exec_ax(self, command: str, timeout: int = 120) -> ExecResult:
         """Async version of exec_ax — see MacOSSandbox.exec_ax for details."""
         resp = await self._http.post(
-            f"{self._prefix}/exec_ax", json={"command": command}, timeout=timeout
+            f"{self._prefix}/exec_ax",
+            json={"command": command, "timeout": timeout},
+            timeout=timeout,
         )
         resp.raise_for_status()
         return ExecResult.from_dict(resp.json())