lm-deluge 0.0.83__py3-none-any.whl → 0.0.85__py3-none-any.whl

lm_deluge/tool/builtin/gemini.py

@@ -0,0 +1,59 @@
+"""
+Gemini built-in tools including computer use.
+
+Gemini computer use works differently from OpenAI/Anthropic:
+- Uses a special ComputerUse tool type in the API request
+- Returns actions as regular function_call objects
+- Uses normalized coordinates (0-999) that must be denormalized
+- Function responses include screenshots as FunctionResponsePart
+"""
+
+from typing import Literal
+
+
+def computer_use_gemini(
+    environment: Literal["browser", "android"] = "browser",
+    excluded_functions: list[str] | None = None,
+) -> dict:
+    """
+    Create a Gemini computer use tool configuration.
+
+    This returns a dict that will be specially handled when building
+    the Gemini API request.
+
+    Args:
+        environment: The environment type - "browser" or "android"
+        excluded_functions: List of predefined function names to exclude.
+            Available functions:
+            - open_web_browser, wait_5_seconds, go_back, go_forward
+            - search, navigate, click_at, hover_at, type_text_at
+            - key_combination, scroll_document, scroll_at, drag_and_drop
+
+    Returns:
+        A dict that will be converted to ComputerUse tool config
+    """
+    result: dict[str, str | list[str]] = {
+        "type": "gemini_computer_use",
+        "environment": environment,
+    }
+    if excluded_functions:
+        result["excluded_predefined_functions"] = excluded_functions
+    return result
+
+
+# Constants for Gemini computer use action names
+GEMINI_CU_ACTIONS = [
+    "open_web_browser",
+    "wait_5_seconds",
+    "go_back",
+    "go_forward",
+    "search",
+    "navigate",
+    "click_at",
+    "hover_at",
+    "type_text_at",
+    "key_combination",
+    "scroll_document",
+    "scroll_at",
+    "drag_and_drop",
+]

lm_deluge/tool/builtin/openai.py

@@ -0,0 +1,74 @@
+def image_generation_openai():
+    # TODO: handle result properly
+    return {"type": "image_generation"}
+
+
+def code_interpreter_openai(container: dict | None = None):
+    if container is None:
+        container = {"type": "auto"}
+    return {"type": "code_interpreter", "container": container}
+
+
+def local_shell_openai():
+    return {"type": "local_shell"}
+
+
+def web_search_openai(
+    preview: bool = False,
+    user_location: dict | None = None,
+    allowed_domains: list[str] | None = None,
+    search_context_size: str | None = None,
+):
+    """OpenAI's built-in web search tool for the Responses API.
+
+    Args:
+        preview: If True, use web_search_preview. If False (default), use
+            the GA web_search tool.
+        user_location: Optional approximate user location to refine search results.
+            Should be a dict with "type": "approximate" and an "approximate" key
+            containing any of: country (ISO code), city, region, timezone.
+            Note: Not supported for deep research models.
+        allowed_domains: Optional list of domains to restrict search results to.
+            Up to 100 URLs, without http/https prefix (e.g. "openai.com").
+            Only available with web_search (not preview).
+        search_context_size: Controls how much context from web search results
+            is provided to the model. Options: "low", "medium" (default), "high".
+            Higher values use more tokens but may improve response quality.
+
+    Returns:
+        A dict representing the web search tool configuration.
+    """
+    tool: dict = {}
+    if preview:
+        tool["type"] = "web_search_preview"
+        if user_location:
+            tool["user_location"] = user_location
+        if search_context_size:
+            tool["search_context_size"] = search_context_size
+        return tool
+
+    # GA web_search tool
+    tool["type"] = "web_search"
+
+    if user_location:
+        tool["user_location"] = user_location
+
+    if search_context_size:
+        tool["search_context_size"] = search_context_size
+
+    # Domain filtering uses a nested filters structure
+    if allowed_domains:
+        tool["filters"] = {"allowed_domains": allowed_domains}
+
+    return tool
+
+
+def computer_use_openai(
+    display_width: int = 1024, display_height: int = 768, environment: str = "browser"
+):
+    return {
+        "type": "computer_use_preview",
+        "display_width": display_width,
+        "display_height": display_height,
+        "environment": environment,
+    }

lm_deluge/tool/cua/__init__.py

@@ -0,0 +1,173 @@
+"""
+Computer Use Actions (CUA) module.
+
+This module provides a provider-agnostic abstraction for computer use actions
+and executors that can run them on various backends.
+
+Key components:
+- CUAction: Union type of all possible computer use actions
+- ComputerExecutor: Abstract base class for action executors
+- KernelExecutor: Execute actions on Kernel's browser-as-a-service
+- TryCUAExecutor: Execute actions on TryCUA's computer-server (desktop control)
+
+Usage with Kernel (browser):
+    from lm_deluge.tool.cua import (
+        KernelBrowser,
+        KernelExecutor,
+        anthropic_tool_call_to_action,
+    )
+
+    # Create a browser and executor
+    with KernelBrowser() as browser:
+        executor = KernelExecutor(browser.session_id)
+
+        # Convert Anthropic tool call to action
+        action = anthropic_tool_call_to_action(tool_call.arguments)
+
+        # Execute and get result
+        result = executor.execute(action)
+
+Usage with TryCUA (desktop):
+    from lm_deluge.tool.cua import (
+        TryCUAConnection,
+        TryCUAExecutor,
+        Screenshot,
+        Click,
+        Type,
+    )
+
+    # Connect to a TryCUA computer-server
+    with TryCUAConnection("ws://localhost:8000/ws") as conn:
+        executor = TryCUAExecutor(conn)
+
+        # Execute actions
+        result = executor.execute(Screenshot(kind="screenshot"))
+        executor.execute(Click(kind="click", x=100, y=200, button="left"))
+        executor.execute(Type(kind="type", text="Hello!"))
+
+    # Async version
+    async with AsyncTryCUAConnection("ws://localhost:8000/ws") as conn:
+        executor = AsyncTryCUAExecutor(conn)
+        result = await executor.execute(Screenshot(kind="screenshot"))
+"""
+
+from .actions import (
+    Bash,
+    Click,
+    CUAction,
+    CursorPos,
+    DoubleClick,
+    Drag,
+    Edit,
+    GoBack,
+    GoForward,
+    HoldKey,
+    Keypress,
+    MouseDown,
+    MouseUp,
+    Move,
+    Navigate,
+    Scroll,
+    Screenshot,
+    Search,
+    TripleClick,
+    Type,
+    Wait,
+)
+from .base import ComputerExecutor, CUActionResult
+from .base import Screenshot as ScreenshotResult
+from .converters import (
+    anthropic_tool_call_to_action,
+    openai_computer_call_to_action,
+    gemini_function_call_to_action,
+)
+from .batch import create_computer_batch_tool
+
+
+# Lazy imports for optional dependencies
+def __getattr__(name: str):
+    if name in (
+        "KernelBrowser",
+        "KernelExecutor",
+        "AsyncKernelBrowser",
+        "AsyncKernelExecutor",
+    ):
+        from .kernel import (
+            KernelBrowser,
+            KernelExecutor,
+            AsyncKernelBrowser,
+            AsyncKernelExecutor,
+        )
+
+        return {
+            "KernelBrowser": KernelBrowser,
+            "KernelExecutor": KernelExecutor,
+            "AsyncKernelBrowser": AsyncKernelBrowser,
+            "AsyncKernelExecutor": AsyncKernelExecutor,
+        }[name]
+    if name in (
+        "TryCUAConnection",
+        "TryCUAExecutor",
+        "AsyncTryCUAConnection",
+        "AsyncTryCUAExecutor",
+    ):
+        from .trycua import (
+            TryCUAConnection,
+            TryCUAExecutor,
+            AsyncTryCUAConnection,
+            AsyncTryCUAExecutor,
+        )
+
+        return {
+            "TryCUAConnection": TryCUAConnection,
+            "TryCUAExecutor": TryCUAExecutor,
+            "AsyncTryCUAConnection": AsyncTryCUAConnection,
+            "AsyncTryCUAExecutor": AsyncTryCUAExecutor,
+        }[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    # Actions
+    "CUAction",
+    "Click",
+    "DoubleClick",
+    "TripleClick",
+    "Move",
+    "Drag",
+    "Scroll",
+    "Keypress",
+    "Type",
+    "Wait",
+    "Screenshot",
+    "MouseDown",
+    "MouseUp",
+    "CursorPos",
+    "HoldKey",
+    "Navigate",
+    "GoBack",
+    "GoForward",
+    "Search",
+    "Bash",
+    "Edit",
+    # Base classes
+    "ComputerExecutor",
+    "CUActionResult",
+    "ScreenshotResult",
+    # Converters
+    "anthropic_tool_call_to_action",
+    "openai_computer_call_to_action",
+    "gemini_function_call_to_action",
+    # Batch tool
+    "create_computer_batch_tool",
+    # Kernel executor (lazy loaded)
+    "KernelBrowser",  # pyright: ignore[reportUnsupportedDunderAll]
+    "KernelExecutor",  # pyright: ignore[reportUnsupportedDunderAll]
+    "AsyncKernelBrowser",  # pyright: ignore[reportUnsupportedDunderAll]
+    "AsyncKernelExecutor",  # pyright: ignore[reportUnsupportedDunderAll]
+    # TryCUA executor (lazy loaded)
+    "TryCUAConnection",  # pyright: ignore[reportUnsupportedDunderAll]
+    "TryCUAExecutor",  # pyright: ignore[reportUnsupportedDunderAll]
+    "AsyncTryCUAConnection",  # pyright: ignore[reportUnsupportedDunderAll]
+    "AsyncTryCUAExecutor",  # pyright: ignore[reportUnsupportedDunderAll]
+]

lm_deluge/tool/cua/actions.py

@@ -0,0 +1,148 @@
+from typing import Any, List, Literal, TypedDict, Union
+
+Coord = tuple[int, int]
+
+
+class CUActionBase(TypedDict):
+    kind: str | Any  # discriminator
+
+
+class Click(CUActionBase):
+    kind: Literal["click"]
+    x: int | None  # if missing, current cursor position
+    y: int | None
+    button: Literal["left", "right", "middle", "back", "forward"]
+
+
+class DoubleClick(CUActionBase):
+    kind: Literal["double_click"]
+    x: int | None  # if missing, current cursor position
+    y: int | None
+
+
+class Move(CUActionBase):
+    kind: Literal["move"]
+    x: int
+    y: int
+
+
+class Drag(CUActionBase):
+    kind: Literal["drag"]
+    start_x: int | None  # if missing, current cursor position
+    start_y: int | None  # if missing, current cursor position
+    path: List[Coord]  # path to drag after mousedown
+
+
+class Scroll(CUActionBase):
+    kind: Literal["scroll"]
+    x: int | None  # if not provided, current cursor position
+    y: int | None  # if not provided, current cursor position
+    dx: int  # scroll_x in OpenAI
+    dy: int  # scroll_y in OpenAI
+
+
+class Keypress(CUActionBase):
+    kind: Literal["keypress"]
+    keys: List[str]
+
+
+class Type(CUActionBase):
+    kind: Literal["type"]
+    text: str
+
+
+class Wait(CUActionBase):
+    kind: Literal["wait"]
+    ms: int
+
+
+class Screenshot(CUActionBase):
+    kind: Literal["screenshot"]
+
+
+class MouseDown(CUActionBase):
+    kind: Literal["mouse_down"]
+    button: Literal["left", "right", "middle", "back", "forward"]
+
+
+class MouseUp(CUActionBase):
+    kind: Literal["mouse_up"]
+    button: Literal["left", "right", "middle", "back", "forward"]
+
+
+class CursorPos(CUActionBase):
+    kind: Literal["cursor_position"]
+
+
+class HoldKey(CUActionBase):
+    kind: Literal["hold_key"]
+    key: str
+    ms: int  # duration
+
+
+class TripleClick(CUActionBase):
+    kind: Literal["triple_click"]
+    x: int | None  # if missing, current cursor position
+    y: int | None
+
+
+# ── Browser‑level actions ────────────────────────────────────────────
+class Navigate(CUActionBase):
+    kind: Literal["navigate"]
+    url: str
+
+
+class GoBack(CUActionBase):
+    kind: Literal["go_back"]
+
+
+class GoForward(CUActionBase):
+    kind: Literal["go_forward"]
+
+
+class Search(CUActionBase):
+    kind: Literal["search"]
+    query: str
+
+
+# ── Bash / Editor (provider‑independent) ────────────────────────────
+class Bash(CUActionBase):
+    kind: Literal["bash"]
+    command: str | None
+    restart: bool | None
+
+
+class Edit(CUActionBase):
+    kind: Literal["edit"]
+    command: Literal["view", "create", "str_replace", "insert", "undo_edit"]
+    path: str
+    # optional, keep names identical to Anthropic spec
+    file_text: str | None
+    view_range: List[int] | None
+    old_str: str | None
+    new_str: str | None
+    insert_line: int | None
+
+
+CUAction = Union[
+    Click,
+    DoubleClick,
+    TripleClick,
+    MouseDown,
+    MouseUp,
+    Drag,
+    Move,
+    Scroll,
+    Keypress,
+    Type,
+    HoldKey,
+    Wait,
+    Screenshot,
+    CursorPos,
+    Navigate,
+    GoBack,
+    GoForward,
+    Search,
+    Bash,
+    Edit,
+]

lm_deluge/tool/cua/base.py

@@ -0,0 +1,27 @@
+import abc
+from typing import TypedDict
+
+from .actions import CUAction
+
+
+class Screenshot(TypedDict):
+    media_type: str
+    content: bytes
+
+
+class CUActionResult(TypedDict):
+    screenshot: Screenshot | None
+    data: dict  # for structured metadata
+
+
+class ComputerExecutor(abc.ABC):
+    """
+    A computer executor is any class that can take an action (from actions.py)
+    and "execute" it. This allows us to plug any API provider (OpenAI, Anthropic)
+    into any computer-use backend (BrowserBase, Kernel, Modal sandbox) by:
+    - Mapping each provider's tools to some (sub)set of CUActions
+    - Defining how to run each CUAction on that backend
+    """
+
+    def execute(self, action: CUAction) -> CUActionResult:
+        raise NotImplementedError("Subclasses must implement execute method")

lm_deluge/tool/cua/batch.py

@@ -0,0 +1,215 @@
+"""
+Batch tool for computer use actions.
+
+Allows Claude to submit multiple computer actions in a single tool call,
+executing them sequentially and returning only one screenshot at the end.
+This dramatically reduces roundtrips for common action sequences like:
+- Ctrl+L → type URL → Return → wait → screenshot
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Any
+
+from .. import Tool
+from .converters import anthropic_tool_call_to_action
+
+
+# Define the action schema matching Anthropic's computer tool
+ACTION_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "action": {
+            "type": "string",
+            "enum": [
+                "screenshot",
+                "key",
+                "type",
+                "mouse_move",
+                "left_click",
+                "left_click_drag",
+                "right_click",
+                "middle_click",
+                "double_click",
+                "triple_click",
+                "scroll",
+                "wait",
+                "cursor_position",
+            ],
+            "description": "The action to perform",
+        },
+        "text": {
+            "type": "string",
+            "description": "For 'key' action: key combo like 'Return', 'ctrl+l'. For 'type' action: text to type.",
+        },
+        "coordinate": {
+            "type": "array",
+            "items": {"type": "integer"},
+            "minItems": 2,
+            "maxItems": 2,
+            "description": "For click/move actions: [x, y] coordinates",
+        },
+        "scroll_direction": {
+            "type": "string",
+            "enum": ["up", "down", "left", "right"],
+            "description": "For scroll action: direction to scroll",
+        },
+        "scroll_amount": {
+            "type": "integer",
+            "description": "For scroll action: number of scroll clicks",
+        },
+        "duration": {
+            "type": "number",
+            "description": "For wait action: seconds to wait",
+        },
+    },
+    "required": ["action"],
+}
+
+
+def create_computer_batch_tool(
+    executor,  # AsyncKernelExecutor or similar
+    *,
+    tool_name: str = "computer_batch",
+    include_final_screenshot: bool = True,
+) -> Tool:
+    """
+    Create a batch tool for computer use actions.
+
+    This tool allows Claude to submit multiple actions in one call:
+    - Actions execute sequentially
+    - Only one screenshot is returned at the end (if requested)
+    - Dramatically reduces API roundtrips
+
+    Args:
+        executor: The computer executor (e.g., AsyncKernelExecutor)
+        tool_name: Name for the batch tool
+        include_final_screenshot: Whether to always include a screenshot at the end
+
+    Returns:
+        A Tool that can be passed to the LLM
+
+    Example:
+        executor = AsyncKernelExecutor(session_id)
+        batch_tool = create_computer_batch_tool(executor)
+
+        # Claude can now call:
+        # computer_batch(actions=[
+        #     {"action": "key", "text": "ctrl+l"},
+        #     {"action": "type", "text": "https://example.com"},
+        #     {"action": "key", "text": "Return"},
+        #     {"action": "wait", "duration": 2},
+        #     {"action": "screenshot"}
+        # ])
+    """
+
+    async def run_batch(actions: list[dict[str, Any]]) -> str | list:
+        """Execute a batch of computer actions and return results."""
+        from ...image import Image
+        from ...prompt import Text
+
+        results = []
+        final_screenshot = None
+
+        for i, action_args in enumerate(actions):
+            action_name = action_args.get("action", "unknown")
+
+            try:
+                # Convert Anthropic format to CUAction
+                cu_action = anthropic_tool_call_to_action(action_args)
+
+                # Execute the action
+                result = await executor.execute(cu_action)
+
+                # Track if this was a screenshot
+                if result.get("screenshot"):
+                    final_screenshot = result["screenshot"]
+                    results.append(
+                        {
+                            "action": action_name,
+                            "status": "ok",
+                            "has_screenshot": True,
+                        }
+                    )
+                else:
+                    results.append(
+                        {
+                            "action": action_name,
+                            "status": "ok",
+                        }
+                    )
+
+            except Exception as e:
+                results.append(
+                    {
+                        "action": action_name,
+                        "status": "error",
+                        "error": str(e),
+                    }
+                )
+                # Stop on error
+                break
+
+        # If we should include a final screenshot and don't have one yet, take one
+        if include_final_screenshot and final_screenshot is None:
+            try:
+                from .actions import Screenshot
+
+                result = await executor.execute(Screenshot(kind="screenshot"))
+                if result.get("screenshot"):
+                    final_screenshot = result["screenshot"]
+            except Exception:
+                pass
+
+        # Build the response
+        summary = f"Executed {len(results)} actions. "
+        errors = [r for r in results if r.get("status") == "error"]
+        if errors:
+            summary += f"{len(errors)} failed: {errors[0].get('error', 'unknown')}"
+        else:
+            summary += "All succeeded."
+
+        if final_screenshot:
+            # Return Text + Image (proper ToolResultPart types)
+            screenshot_bytes = final_screenshot["content"]
+            b64 = base64.b64encode(screenshot_bytes).decode()
+            img = Image(data=f"data:image/png;base64,{b64}")
+            return [Text(summary), img]
+        else:
+            # Just return text summary
+            return summary
+
+    description = """Execute multiple computer actions in a single call.
+This is much faster than calling actions one at a time.
+Actions run sequentially. A screenshot is taken at the end.
+
+Common patterns:
+- Navigate to URL: [{"action":"key","text":"ctrl+l"}, {"action":"type","text":"https://..."}, {"action":"key","text":"Return"}, {"action":"wait","duration":2}]
+- Click and type: [{"action":"left_click","coordinate":[x,y]}, {"action":"type","text":"..."}]
+- Scroll and screenshot: [{"action":"scroll","coordinate":[x,y],"scroll_direction":"down","scroll_amount":3}]
+
+Available actions:
+- screenshot: Capture the screen
+- key: Press key combo (text="Return", "ctrl+l", "ctrl+a", etc.)
+- type: Type text (text="hello world")
+- left_click, right_click, middle_click, double_click, triple_click: Click at coordinate=[x,y]
+- mouse_move: Move cursor to coordinate=[x,y]
+- scroll: Scroll at coordinate=[x,y] with scroll_direction and scroll_amount
+- wait: Pause for duration seconds
+"""
+
+    return Tool(
+        name=tool_name,
+        description=description,
+        parameters={
+            "actions": {
+                "type": "array",
+                "description": "List of actions to execute in order",
+                "items": ACTION_SCHEMA,
+                "minItems": 1,
+            }
+        },
+        required=["actions"],
+        run=run_batch,
+    )