use-computer 0.0.39__tar.gz → 0.0.41__tar.gz

{use_computer-0.0.39 → use_computer-0.0.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: use-computer
-Version: 0.0.39
+Version: 0.0.41
 Summary: Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu
 Project-URL: Homepage, https://use.computer
 Project-URL: Documentation, https://api.use.computer/docs
@@ -24,6 +24,8 @@ Requires-Dist: google-genai>=1; extra == 'agents'
 Requires-Dist: litellm>=1; extra == 'agents'
 Requires-Dist: openai>=1; extra == 'agents'
 Requires-Dist: pillow>=10; extra == 'agents'
+Requires-Dist: tinker-cookbook>=0.1.0; (python_full_version >= '3.11') and extra == 'agents'
+Requires-Dist: tinker>=0.14.0; (python_full_version >= '3.11') and extra == 'agents'
 Provides-Extra: harbor
 Requires-Dist: harbor>=0.4.0; (python_version >= '3.12') and extra == 'harbor'
 Requires-Dist: pyyaml>=6; extra == 'harbor'

{use_computer-0.0.39 → use_computer-0.0.41}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "use-computer"
-version = "0.0.39"
+version = "0.0.41"
 description = "Python SDK for use.computer sandboxes across macOS, iOS simulators, Windows, and Ubuntu"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -30,6 +30,8 @@ agents = [
     "anthropic>=0.86",
     "litellm>=1",
     "google-genai>=1",
+    "tinker>=0.14.0 ; python_full_version >= '3.11'",
+    "tinker-cookbook>=0.1.0 ; python_full_version >= '3.11'",
 ]
 harbor = [
     "harbor>=0.4.0; python_version >= '3.12'",

{use_computer-0.0.39 → use_computer-0.0.41}/scripts/smoke_wheel.py

@@ -102,7 +102,7 @@ with tempfile.TemporaryDirectory() as tmp:
 
 for agent_cls, model_name in [
     (AnthropicComputerAgent, "anthropic/claude-sonnet-4-6"),
-    (OpenAIComputerAgent, "openai/computer-use-preview"),
+    (OpenAIComputerAgent, "openai/gpt-5.5"),
     (GeminiComputerAgent, "gemini/gemini-2.5-computer-use-preview-10-2025"),
     (GenericComputerAgent, "openai/gpt-4o"),
     (IOSComputerAgent, "anthropic/claude-sonnet-4-6"),

{use_computer-0.0.39 → use_computer-0.0.41}/use_computer/agents/base/actions.py

@@ -140,6 +140,7 @@ async def execute_action(
             await sandbox.keyboard.type(command.text)
         elif command.action is DesktopAction.KEY:
             key = normalize_key(command.key or command.text, key_aliases)
+            action["key"] = key
             if "+" in key:
                 await sandbox.keyboard.hotkey(key)
             else:

{use_computer-0.0.39 → use_computer-0.0.41}/use_computer/agents/base/desktop.py

@@ -12,32 +12,102 @@ from use_computer.agents.types import Platform
 _KEY_ALIASES = {
     "Escape": "esc",
     "escape": "esc",
+    "ESCAPE": "esc",
+    "ESC": "esc",
     "Return": "enter",
     "return": "enter",
+    "RETURN": "enter",
+    "Enter": "enter",
+    "enter": "enter",
+    "ENTER": "enter",
+    "Command": "cmd",
+    "command": "cmd",
+    "COMMAND": "cmd",
+    "Cmd": "cmd",
+    "cmd": "cmd",
+    "CMD": "cmd",
     "super": "cmd",
+    "SUPER": "cmd",
     "meta": "cmd",
+    "META": "cmd",
     "Super_L": "cmd",
     "Meta_L": "cmd",
     "Control": "ctrl",
+    "control": "ctrl",
+    "CONTROL": "ctrl",
+    "Ctrl": "ctrl",
+    "ctrl": "ctrl",
+    "CTRL": "ctrl",
     "Shift": "shift",
+    "shift": "shift",
+    "SHIFT": "shift",
     "Alt": "alt",
+    "alt": "alt",
+    "ALT": "alt",
+    "Option": "alt",
+    "option": "alt",
+    "OPTION": "alt",
     "Backspace": "backspace",
+    "backspace": "backspace",
+    "BACKSPACE": "backspace",
     "Delete": "delete",
+    "delete": "delete",
+    "DELETE": "delete",
+    "Tab": "tab",
+    "tab": "tab",
+    "TAB": "tab",
+    "Space": "space",
+    "space": "space",
+    "SPACE": "space",
     "ArrowUp": "up",
+    "ARROWUP": "up",
+    "Up": "up",
+    "up": "up",
+    "UP": "up",
     "ArrowDown": "down",
+    "ARROWDOWN": "down",
+    "Down": "down",
+    "down": "down",
+    "DOWN": "down",
     "ArrowLeft": "left",
+    "ARROWLEFT": "left",
+    "Left": "left",
+    "left": "left",
+    "LEFT": "left",
     "ArrowRight": "right",
+    "ARROWRIGHT": "right",
+    "Right": "right",
+    "right": "right",
+    "RIGHT": "right",
     "Page_Up": "pageup",
+    "PAGE_UP": "pageup",
+    "PAGEUP": "pageup",
     "Page_Down": "pagedown",
+    "PAGE_DOWN": "pagedown",
+    "PAGEDOWN": "pagedown",
 }
 
 # Windows: the modifier is the Windows key, not Cmd.
 _WINDOWS_KEY_ALIASES = {
     **_KEY_ALIASES,
+    "Command": "win",
+    "command": "win",
+    "COMMAND": "win",
+    "Cmd": "win",
+    "cmd": "win",
+    "CMD": "win",
     "super": "win",
+    "SUPER": "win",
     "meta": "win",
+    "META": "win",
     "Super_L": "win",
     "Meta_L": "win",
+    "Win": "win",
+    "win": "win",
+    "WIN": "win",
+    "Windows": "win",
+    "windows": "win",
+    "WINDOWS": "win",
 }
 
 

{use_computer-0.0.39 → use_computer-0.0.41}/use_computer/agents/base/recordings.py

@@ -11,6 +11,11 @@ from use_computer.sandbox import AsyncSandbox
 
 class RecordingMixin:
     async def start_recording(self: Any, sandbox: AsyncSandbox) -> None:
+        if getattr(getattr(sandbox, "type", None), "value", "") == "ios":
+            try:
+                await sandbox.screenshot.take_full_screen()
+            except Exception as e:
+                self.logger.warning(f"iOS recording warm-up screenshot failed: {e}")
         try:
             result = await sandbox.recording.start(name="trial")
             self._recording_id = result.id
@@ -22,6 +27,7 @@ class RecordingMixin:
         if not self._recording_id:
             return
         await asyncio.sleep(3.0)
+        rec_path = self.logs_dir / "recording.mp4"
         try:
             info = await sandbox.recording.stop(self._recording_id)
             self.logger.info(
@@ -33,16 +39,24 @@ class RecordingMixin:
 
             await asyncio.sleep(2)
 
-            rec_path = self.logs_dir / "recording.mp4"
-            self.logger.info(f"Downloading recording → {rec_path}")
-            t_dl = time.monotonic()
-            await sandbox.recording.download(self._recording_id, str(rec_path), timeout=180.0)
-            dl_elapsed = time.monotonic() - t_dl
-            local_size = rec_path.stat().st_size
-            self.logger.info(
-                f"Recording saved: {rec_path} "
-                f"local_size={local_size}B ({local_size / (1024 * 1024):.2f} MB) "
-                f"download_elapsed={dl_elapsed:.1f}s"
-            )
+            await self._download_recording(sandbox, rec_path)
         except Exception as e:
             self.logger.warning(f"Recording save failed: {type(e).__name__}: {e}")
+            try:
+                await self._download_recording(sandbox, rec_path)
+            except Exception as fallback:
+                self.logger.warning(
+                    f"Recording fallback download failed: {type(fallback).__name__}: {fallback}"
+                )
+
+    async def _download_recording(self: Any, sandbox: AsyncSandbox, rec_path) -> None:
+        self.logger.info(f"Downloading recording → {rec_path}")
+        t_dl = time.monotonic()
+        await sandbox.recording.download(self._recording_id, str(rec_path), timeout=180.0)
+        dl_elapsed = time.monotonic() - t_dl
+        local_size = rec_path.stat().st_size
+        self.logger.info(
+            f"Recording saved: {rec_path} "
+            f"local_size={local_size}B ({local_size / (1024 * 1024):.2f} MB) "
+            f"download_elapsed={dl_elapsed:.1f}s"
+        )

{use_computer-0.0.39 → use_computer-0.0.41}/use_computer/agents/providers/generic.py

@@ -83,6 +83,11 @@ class GenericCUAAgent(BaseCUAAgent):
         coord_mode: str | None = None,
         reasoning_effort: str | None = None,
         reasoning_history: str | None = None,
+        renderer_name: str | None = None,
+        model_path: str | None = None,
+        temperature: float = 1.0,
+        top_p: float = 0.95,
+        top_k: int = -1,
         **kwargs: Any,
     ):
         super().__init__(logs_dir=logs_dir, model_name=model_name, **kwargs)
@@ -106,7 +111,12 @@ class GenericCUAAgent(BaseCUAAgent):
                 ReasoningHistory(reasoning_history) if reasoning_history is not None else None
             ),
         ).as_dict()
-        self._tinker_llm = None
+        self._tinker_backend = None
+        self._renderer_name = renderer_name
+        self._model_path = model_path
+        self._temperature = temperature
+        self._top_p = top_p
+        self._top_k = top_k
         # Coordinate convention emitted by the model:
         #   "api_pixel"        — Kimi default: 0..1 floats OR pixel ints in API space (1280x800)
         #   "normalized_1000"  — Qwen-VL default: 0..1000 ints as per-mille of screen
@@ -168,39 +178,19 @@ class GenericCUAAgent(BaseCUAAgent):
     async def _tinker_completion(
         self, model: str, messages: list[dict], max_tokens: int
     ) -> tuple[str, int, int]:
-        if self._tinker_llm is None:
-            from harbor.llms.tinker import TinkerLLM
+        if self._tinker_backend is None:
+            from use_computer.agents.providers.tinker_backend import TinkerBackend
 
-            self._tinker_llm = TinkerLLM(
+            self._tinker_backend = TinkerBackend(
                 model_name=model,
+                model_path=self._model_path,
+                renderer_name=self._renderer_name,
+                temperature=self._temperature,
+                top_p=self._top_p,
+                top_k=self._top_k,
                 max_tokens=max_tokens,
             )
-
-        # TinkerLLM.call() expects (prompt, message_history)
-        # Split system + history from the final user message
-        history = messages[:-1] if len(messages) > 1 else []
-        prompt = ""
-        last_msg = messages[-1] if messages else {"content": ""}
-        content = last_msg.get("content")
-        if isinstance(content, str):
-            prompt = content
-        elif isinstance(content, list):
-            prompt = " ".join(
-                p.get("text", "")
-                for p in content
-                if isinstance(p, dict) and p.get("type") == "text"
-            )
-
-        response = await self._tinker_llm.call(prompt=prompt, message_history=history)
-        content = response.content or ""
-        text = (
-            content
-            if isinstance(content, str)
-            else "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in content)
-        )
-        prompt_tokens = response.usage.prompt_tokens if response.usage else 0
-        completion_tokens = response.usage.completion_tokens if response.usage else 0
-        return text, prompt_tokens, completion_tokens
+        return await self._tinker_backend.complete(messages, max_tokens)
 
     async def _run_agent(
         self,

{use_computer-0.0.39 → use_computer-0.0.41}/use_computer/agents/providers/openai.py

@@ -19,6 +19,9 @@ from use_computer.agents.base import (
 )
 from use_computer.agents.types import DesktopAction, ScrollDirection
 
+DEFAULT_OPENAI_COMPUTER_MODEL = "gpt-5.5"
+LEGACY_OPENAI_COMPUTER_MODEL = "computer-use-preview"
+
 
 class OpenAIToolType(str, Enum):
     COMPUTER = "computer"
@@ -31,6 +34,7 @@ class OpenAIItemType(str, Enum):
     COMPUTER_SCREENSHOT = "computer_screenshot"
     INPUT_IMAGE = "input_image"
     MESSAGE = "message"
+    OUTPUT_TEXT = "output_text"
 
 
 @dataclass(frozen=True)
@@ -46,7 +50,6 @@ class OpenAIComputerObservation:
             "output": {
                 "type": OpenAIItemType.COMPUTER_SCREENSHOT.value,
                 "image_url": f"data:image/png;base64,{self.screenshot_b64}",
-                "detail": "original",
             },
         }
         if self.pending_safety_checks:
@@ -69,7 +72,7 @@ class OpenAICUAAgent(BaseCUAAgent):
         self.steps[0]["message"] = instruction
 
         client = openai.OpenAI()
-        model = self._parsed_model_name or "computer-use-preview"
+        model = self._parsed_model_name or DEFAULT_OPENAI_COMPUTER_MODEL
 
         system = build_system_prompt(
             load_prompt("openai.txt"),
@@ -109,21 +112,14 @@ class OpenAICUAAgent(BaseCUAAgent):
 
         for step_idx in range(self.max_steps):
             await self._fire_in_process(environment, step_idx)
-            if previous_response_id is None:
-                response = client.responses.create(
-                    model=model,
-                    tools=tools,  # type: ignore
-                    input=input_items,
-                    truncation="auto",
-                )
-            else:
-                response = client.responses.create(
-                    model=model,
-                    tools=tools,  # type: ignore
-                    input=input_items,
+            response = client.responses.create(
+                **_response_create_kwargs(
+                    model,
+                    tools=tools,
+                    input_items=input_items,
                     previous_response_id=previous_response_id,
-                    truncation="auto",
                 )
+            )
             previous_response_id = response.id
 
             self.total_in += response.usage.input_tokens if response.usage else 0
@@ -136,16 +132,11 @@ class OpenAICUAAgent(BaseCUAAgent):
             ]
 
             if not computer_calls:
-                text_parts = [
-                    getattr(item, "text", "")
-                    for item in response.output
-                    if getattr(item, "type", None) == OpenAIItemType.MESSAGE.value
-                ]
                 self.steps.append(
                     {
                         "step_id": len(self.steps) + 1,
                         "source": "agent",
-                        "message": "\n".join(text_parts) or "Task complete.",
+                        "message": _response_text(response) or "Task complete.",
                     }
                 )
                 break
@@ -222,7 +213,7 @@ def _openai_tools_for_model(
     height: int,
     environment: str,
 ) -> list[dict[str, Any]]:
-    if model == "computer-use-preview":
+    if _uses_legacy_preview_tool(model):
         return [
             {
                 "type": OpenAIToolType.COMPUTER_USE_PREVIEW.value,
@@ -234,6 +225,29 @@ def _openai_tools_for_model(
     return [{"type": OpenAIToolType.COMPUTER.value}]
 
 
+def _response_create_kwargs(
+    model: str,
+    *,
+    tools: list[dict[str, Any]],
+    input_items: list[Any],
+    previous_response_id: str | None,
+) -> dict[str, Any]:
+    kwargs: dict[str, Any] = {
+        "model": model,
+        "tools": tools,
+        "input": input_items,
+    }
+    if previous_response_id is not None:
+        kwargs["previous_response_id"] = previous_response_id
+    if _uses_legacy_preview_tool(model):
+        kwargs["truncation"] = "auto"
+    return kwargs
+
+
+def _uses_legacy_preview_tool(model: str) -> bool:
+    return model in {LEGACY_OPENAI_COMPUTER_MODEL, "computer-use-preview-2025-03-11"}
+
+
 def _openai_actions(call: Any) -> list[Any]:
     actions = getattr(call, "actions", None)
     if isinstance(actions, Iterable) and not isinstance(actions, (str, bytes)):
@@ -255,8 +269,31 @@ def _pending_safety_checks(call: Any) -> tuple[dict[str, Any], ...]:
     return tuple(normalized)
 
 
+def _response_text(response: Any) -> str:
+    output_text = getattr(response, "output_text", "")
+    if isinstance(output_text, str) and output_text:
+        return output_text
+
+    text_parts = []
+    for item in getattr(response, "output", []):
+        if getattr(item, "type", None) != OpenAIItemType.MESSAGE.value:
+            continue
+        content = getattr(item, "content", None)
+        if isinstance(content, Iterable) and not isinstance(content, (str, bytes)):
+            for block in content:
+                if getattr(block, "type", None) == OpenAIItemType.OUTPUT_TEXT.value:
+                    text = getattr(block, "text", "")
+                    if text:
+                        text_parts.append(text)
+        else:
+            text = getattr(item, "text", "")
+            if text:
+                text_parts.append(text)
+    return "\n".join(text_parts)
+
+
 def _map_openai_action(action: Any) -> dict[str, Any]:
-    """Map OpenAI computer_use_preview action to our standard format."""
+    """Map OpenAI computer actions to our standard format."""
     action_type = action.type
 
     type_map = {

use_computer-0.0.41/use_computer/agents/providers/tinker_backend.py

@@ -0,0 +1,163 @@
+"""Image-aware Tinker sampling backend for the generic CUA agent.
+
+harbor's TinkerLLM forwards text only, so a vision agent loses its screenshots
+on the tinker path. This backend builds the generation prompt through a
+tinker-cookbook renderer with the model's image processor attached, so the
+OpenAI-style `image_url` parts the generic agent emits reach the model.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import importlib
+import io
+from typing import Any, cast
+
+from PIL import Image
+
+
+class TinkerBackend:
+    """Vision-capable tinker sampler that consumes OpenAI chat messages."""
+
+    def __init__(
+        self,
+        *,
+        model_name: str,
+        model_path: str | None = None,
+        renderer_name: str | None = None,
+        temperature: float = 1.0,
+        top_p: float = 0.95,
+        top_k: int = -1,
+        max_tokens: int = 2048,
+        retry_times: int = 5,
+        retry_interval: float = 5.0,
+    ) -> None:
+        self.model_name = model_name
+        self.model_path = model_path
+        self.renderer_name = renderer_name
+        self.temperature = temperature
+        self.top_p = top_p
+        self.top_k = top_k
+        self.max_tokens = max_tokens
+        self.retry_times = retry_times
+        self.retry_interval = retry_interval
+        self._sampling_client = None
+        self._renderer = None
+
+    async def _ensure_ready(self) -> None:
+        if self._sampling_client is not None and self._renderer is not None:
+            return
+        try:
+            tinker = importlib.import_module("tinker")
+            get_image_processor = importlib.import_module(
+                "tinker_cookbook.image_processing_utils"
+            ).get_image_processor
+            get_model_attributes = importlib.import_module(
+                "tinker_cookbook.model_info"
+            ).get_model_attributes
+            get_renderer = importlib.import_module("tinker_cookbook.renderers").get_renderer
+            get_tokenizer = importlib.import_module("tinker_cookbook.tokenizer_utils").get_tokenizer
+        except ImportError as exc:
+            raise ImportError(
+                "TinkerBackend requires `tinker` and `tinker-cookbook`. "
+                "Install them with `uv pip install harbor[tinker]`."
+            ) from exc
+
+        service_client = tinker.ServiceClient()
+        if self.model_path:
+            self._sampling_client = await service_client.create_sampling_client_async(
+                model_path=self.model_path
+            )
+        else:
+            self._sampling_client = await service_client.create_sampling_client_async(
+                base_model=self.model_name
+            )
+
+        tokenizer = get_tokenizer(self.model_name)
+        attrs = get_model_attributes(self.model_name)
+        image_processor = get_image_processor(self.model_name) if attrs.is_vl else None
+        renderer_name = self.renderer_name or _default_renderer(self.model_name)
+        self._renderer = get_renderer(
+            renderer_name,
+            tokenizer,
+            image_processor=image_processor,
+            model_name=self.model_name,
+        )
+
+    async def complete(
+        self, messages: list[dict[str, Any]], max_tokens: int
+    ) -> tuple[str, int, int]:
+        """Sample one response; returns (text, prompt_tokens, completion_tokens)."""
+        await self._ensure_ready()
+        assert self._sampling_client is not None
+        assert self._renderer is not None
+
+        tinker = importlib.import_module("tinker")
+        get_text_content = importlib.import_module("tinker_cookbook.renderers").get_text_content
+
+        model_input = self._renderer.build_generation_prompt(
+            cast(Any, _to_tinker_messages(messages))
+        )
+        stop = self._renderer.get_stop_sequences()
+
+        last_error: Exception | None = None
+        for attempt in range(self.retry_times):
+            try:
+                response = await self._sampling_client.sample_async(
+                    prompt=model_input,
+                    num_samples=1,
+                    sampling_params=tinker.SamplingParams(
+                        temperature=self.temperature,
+                        top_p=self.top_p,
+                        top_k=self.top_k,
+                        max_tokens=max_tokens,
+                        stop=stop,
+                    ),
+                )
+                sequence = response.sequences[0]
+                parsed_message, _termination = self._renderer.parse_response(sequence.tokens)
+                return get_text_content(parsed_message), model_input.length, len(sequence.tokens)
+            except Exception as exc:
+                last_error = exc
+                if attempt == self.retry_times - 1:
+                    break
+                await asyncio.sleep(self.retry_interval)
+        assert last_error is not None
+        raise last_error
+
+
+def _to_tinker_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Convert OpenAI chat messages to tinker-cookbook messages (text + PIL images)."""
+    converted: list[dict[str, Any]] = []
+    for message in messages:
+        content = message.get("content")
+        if isinstance(content, str):
+            converted.append({"role": message["role"], "content": content})
+            continue
+
+        parts: list[dict[str, Any]] = []
+        for part in content or []:
+            if part.get("type") == "text":
+                parts.append({"type": "text", "text": part.get("text", "")})
+            elif part.get("type") == "image_url":
+                url = part.get("image_url", {}).get("url", "")
+                b64 = url.split(",", 1)[1] if "," in url else url
+                image = Image.open(io.BytesIO(base64.b64decode(b64))).convert("RGB")
+                parts.append({"type": "image", "image": image})
+        if len(parts) == 1 and parts[0]["type"] == "text":
+            converted.append({"role": message["role"], "content": parts[0]["text"]})
+        else:
+            converted.append({"role": message["role"], "content": parts})
+    return converted
+
+
+def _default_renderer(model_name: str) -> str:
+    get_recommended_renderer_names = importlib.import_module(
+        "tinker_cookbook.model_info"
+    ).get_recommended_renderer_names
+
+    recommended = get_recommended_renderer_names(model_name)
+    if "qwen3_5_disable_thinking" in recommended:
+        return "qwen3_5_disable_thinking"
+    return recommended[0]