tau-coding-agent 0.7.6__tar.gz → 0.7.7__tar.gz

{tau_coding_agent-0.7.6/tau_coding_agent.egg-info → tau_coding_agent-0.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tau-coding-agent
-Version: 0.7.6
+Version: 0.7.7
 Summary: A self-extensible agent CLI with terminal UI, multi-provider LLM support, session management with branching, and a plugin system for tools and commands
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{tau_coding_agent-0.7.6 → tau_coding_agent-0.7.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tau-coding-agent"
-version = "0.7.6"
+version = "0.7.7"
 description = "A self-extensible agent CLI with terminal UI, multi-provider LLM support, session management with branching, and a plugin system for tools and commands"
 readme = "README.md"
 requires-python = ">=3.12"

tau_coding_agent-0.7.7/tau/builtins/extensions/computer_use/__init__.py

@@ -0,0 +1,66 @@
+"""computer_use — control the local desktop (macOS Accessibility API / Windows UI Automation).
+
+Registers a single ``computer`` tool that drives the mouse, keyboard, and
+application windows through a platform-neutral ``Desktop`` interface (see
+``types.py``); the concrete implementation is picked at registration time by
+``router.get_desktop_class()`` based on the host OS.
+
+Disabled by default (see manifest.json) — this tool controls the real mouse
+and keyboard, so it should be turned on deliberately via /settings or
+extensions.list settings.
+
+  {
+    "extensions": {
+      "list": [{ "path": "computer_use", "settings": { "enabled": true } }]
+    }
+  }
+
+While the desktop session is open (after a computer action='open' call), a
+compact desktop-state summary — focused/open windows plus the accessible
+interactive elements on screen (see state.py) — is injected ephemerally into
+LLM context at the start of every turn, the same way the todo extension
+re-asserts its live list: via the "context" hook returning
+ContextEventResult(ephemeral_messages=[...]). It is never written to session
+history, so it always reflects the current screen rather than a stale
+snapshot from whenever the model last looked.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .computer import ComputerTool
+from .router import get_desktop_class
+from .state import build_state_message
+
+if TYPE_CHECKING:
+    from tau.extensions.api import ExtensionAPI
+    from tau.extensions.context import ExtensionContext
+
+
+def register(tau: ExtensionAPI) -> None:
+    config = tau.config or {}
+    if not config.get("enabled", False):
+        return
+    try:
+        desktop = get_desktop_class()()
+    except RuntimeError:
+        return  # unsupported platform (e.g. Linux)
+
+    tau.register_tool(ComputerTool(desktop))
+
+    def _inject_desktop_state(_event, _ctx: ExtensionContext):
+        text = build_state_message(desktop)
+        if text is not None:
+            from tau.hooks.engine import ContextEventResult
+            from tau.message.types import UserMessage
+            message = UserMessage.from_text(text)
+            return ContextEventResult(ephemeral_messages=[message])
+        return ContextEventResult(ephemeral_messages=[])
+
+    tau.on("context", _inject_desktop_state)
+
+    @tau.on("extension_unload")
+    async def _close_desktop(_event, _ctx: ExtensionContext) -> None:
+        if desktop.is_open:
+            desktop.close()

tau_coding_agent-0.7.7/tau/builtins/extensions/computer_use/computer.py

@@ -0,0 +1,306 @@
+"""computer — control the local desktop (click, type, scroll, window management, screenshots)."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field, model_validator
+
+from tau.tool.render import call_line
+from tau.tool.types import (
+    AbortSignal,
+    Tool,
+    ToolContext,
+    ToolExecutionUpdateCallback,
+    ToolInvocation,
+    ToolKind,
+    ToolResult,
+)
+
+if TYPE_CHECKING:
+    from .types import Desktop
+
+
+class ComputerAction(str, Enum):
+    """Desktop automation action: open, close, click, type, wait, app, scroll, move, drag, shortcut."""
+
+    open = "open"
+    close = "close"
+    click = "click"
+    type = "type"
+    wait = "wait"
+    app = "app"
+    scroll = "scroll"
+    move = "move"
+    drag = "drag"
+    shortcut = "shortcut"
+
+
+class AppMode(str, Enum):
+    launch = "launch"
+    switch = "switch"
+    resize = "resize"
+    move = "move"
+
+
+class MouseButton(str, Enum):
+    left = "left"
+    right = "right"
+    middle = "middle"
+
+
+class ScrollOrientation(str, Enum):
+    vertical = "vertical"
+    horizontal = "horizontal"
+
+
+class ScrollDirection(str, Enum):
+    up = "up"
+    down = "down"
+    left = "left"
+    right = "right"
+
+
+class CaretPosition(str, Enum):
+    start = "start"
+    idle = "idle"
+    end = "end"
+
+
+class ComputerSchema(BaseModel):
+    """Input schema for computer; action-specific required fields validated via model_validator."""
+
+    action: ComputerAction = Field(
+        description=(
+            "Computer action to perform: open (enable desktop access), "
+            "close (release desktop access), click, type, wait, app, "
+            "scroll, move, drag, or shortcut."
+        )
+    )
+    loc: tuple[int, int] | None = Field(default=None, description="Target screen coordinate as [x, y].")
+    x: int | None = Field(default=None, description="Target x coordinate. Used when loc is omitted.")
+    y: int | None = Field(default=None, description="Target y coordinate. Used when loc is omitted.")
+    text: str | None = Field(default=None, description="Text to type. Required for action=type.")
+    duration: float = Field(default=1.0, ge=0, le=60, description="Seconds to wait for action=wait.")
+    button: MouseButton = Field(default=MouseButton.left, description="Mouse button for action=click.")
+    clicks: int = Field(default=1, ge=0, le=3, description="Number of clicks. Use 0 to only move the pointer.")
+    clear: bool = Field(default=False, description="Clear focused text before typing.")
+    press_enter: bool = Field(default=False, description="Press Enter after typing.")
+    caret_position: CaretPosition = Field(default=CaretPosition.idle, description="Caret movement before typing.")
+    app_mode: AppMode = Field(default=AppMode.launch, description="Application operation for action=app.")
+    name: str | None = Field(default=None, description="Application name for action=app.")
+    size: tuple[int, int] | None = Field(default=None, description="Window size as [width, height] for app resize.")
+    orientation: ScrollOrientation = Field(default=ScrollOrientation.vertical, description="Scroll axis.")
+    direction: ScrollDirection = Field(default=ScrollDirection.down, description="Scroll direction.")
+    wheel_times: int = Field(default=1, ge=1, le=20, description="Number of wheel ticks for action=scroll.")
+    shortcut: str | None = Field(default=None, description="Keyboard shortcut such as command+c or ctrl+c.")
+
+    @model_validator(mode="after")
+    def _check_action_fields(self) -> ComputerSchema:
+        loc = self.loc or ((self.x, self.y) if self.x is not None and self.y is not None else None)
+        if (
+            self.action in {ComputerAction.click, ComputerAction.type, ComputerAction.move, ComputerAction.drag}
+            and loc is None
+        ):
+            raise ValueError("'loc' or both 'x' and 'y' are required for this action")
+        if self.action == ComputerAction.type and self.text is None:
+            raise ValueError("'text' is required when action='type'")
+        if self.action == ComputerAction.shortcut and not self.shortcut:
+            raise ValueError("'shortcut' is required when action='shortcut'")
+        if self.action == ComputerAction.app:
+            if self.app_mode in {AppMode.launch, AppMode.switch} and not self.name:
+                raise ValueError("'name' is required when action='app' with launch or switch")
+            if self.app_mode == AppMode.resize and self.size is None:
+                raise ValueError("'size' is required when action='app' with resize")
+            if self.app_mode == AppMode.move and loc is None:
+                raise ValueError("'loc' or both 'x' and 'y' are required when action='app' with move")
+        return self
+
+
+def _render_call(args: dict, _streaming: bool = False) -> list[str]:
+    action = args.get("action", "")
+    detail = {
+        "type": args.get("text", ""),
+        "app": args.get("name", ""),
+        "shortcut": args.get("shortcut", ""),
+    }.get(action, "")
+    return call_line("computer", action, detail)
+
+
+class ComputerTool(Tool):
+    """Desktop automation tool that drives mouse, keyboard, and application management."""
+
+    def __init__(self, desktop: Desktop) -> None:
+        self._desktop = desktop
+        super().__init__(
+            name="computer",
+            description=(
+                "Control the local desktop through one action-based computer tool. "
+                "Use open to enable desktop access (required before any other action) and "
+                "close to release it. Use click/type/scroll/move/drag for pointer and text input, "
+                "shortcut for keyboard shortcuts, wait for delays, and app for "
+                "launching, switching, resizing, or moving applications."
+            ),
+            schema=ComputerSchema,
+            kind=ToolKind.Execute,
+            render_call=_render_call,
+            prompt_guidelines=(
+                "Call action='open' once before any other computer action, and action='close' "
+                "when finished controlling the desktop. Prefer the dedicated read/write/edit/"
+                "terminal tools over the computer tool for anything achievable without a GUI."
+            ),
+        )
+
+    def get_display_name(self, args: dict) -> str:
+        action = args.get("action", "")
+        text = args.get("text") or ""
+        name = args.get("name") or ""
+        app_mode = args.get("app_mode") or ""
+        shortcut = args.get("shortcut") or ""
+        duration = args.get("duration", "")
+        if action == "open":
+            return "Opening desktop"
+        if action == "close":
+            return "Closing desktop"
+        if action == "click":
+            return "Clicking"
+        if action == "type":
+            return f"Typing: {text[:30]}" if text else "Typing"
+        if action == "wait":
+            return f"Waiting {duration}s" if duration else "Waiting"
+        if action == "app":
+            if app_mode == "launch":
+                return f"Launching: {name}" if name else "Launching app"
+            if app_mode == "switch":
+                return f"Switching to: {name}" if name else "Switching app"
+            return f"App: {app_mode}" if app_mode else "App"
+        if action == "scroll":
+            return "Scrolling"
+        if action == "move":
+            return "Moving cursor"
+        if action == "drag":
+            return "Dragging"
+        if action == "shortcut":
+            return f"Shortcut: {shortcut}" if shortcut else "Shortcut"
+        return "Computer"
+
+    async def execute(
+        self,
+        invocation: ToolInvocation,
+        tool_execution_update_callback: ToolExecutionUpdateCallback | None = None,
+        signal: AbortSignal | None = None,
+        context: ToolContext | None = None,
+    ) -> ToolResult:
+        try:
+            params = ComputerSchema.model_validate(invocation.params)
+        except Exception as exc:
+            return ToolResult.error(id=invocation.id, content=f"computer: {exc}")
+
+        async def update(text: str) -> None:
+            if tool_execution_update_callback is not None:
+                await tool_execution_update_callback(ToolResult.ok(invocation.id, text))
+
+        desktop = self._desktop
+        try:
+            if params.action == ComputerAction.open:
+                if desktop.is_open:
+                    return ToolResult.ok(id=invocation.id, content="Desktop is already open.")
+                await update("Opening desktop…")
+                desktop.open()
+                return ToolResult.ok(id=invocation.id, content="Desktop access enabled.")
+
+            if params.action == ComputerAction.close:
+                if not desktop.is_open:
+                    return ToolResult.ok(id=invocation.id, content="Desktop is not open.")
+                await update("Closing desktop…")
+                desktop.close()
+                return ToolResult.ok(id=invocation.id, content="Desktop access released.")
+
+            if not desktop.is_open:
+                return ToolResult.error(
+                    id=invocation.id,
+                    content="Desktop is not accessible. Use action='open' to enable desktop control first.",
+                )
+
+            await self._announce(update, params)
+            content = self._run_action(desktop, params)
+            return ToolResult.ok(id=invocation.id, content=content)
+        except Exception as exc:
+            return ToolResult.error(id=invocation.id, content=f"computer: {exc}")
+
+    async def _announce(self, update, params: ComputerSchema) -> None:
+        loc = self._loc(params)
+        if params.action == ComputerAction.click:
+            coord = f"({loc[0]}, {loc[1]})" if loc else ""
+            clicks = f" ×{params.clicks}" if params.clicks > 1 else ""
+            await update(f"Clicking {params.button.value}{clicks} at {coord}…")
+        elif params.action == ComputerAction.type:
+            preview = (params.text or "")[:40]
+            preview += "…" if len(params.text or "") > 40 else ""
+            await update(f'Typing "{preview}"…')
+        elif params.action == ComputerAction.wait:
+            await update(f"Waiting {params.duration:g}s…")
+        elif params.action == ComputerAction.app:
+            label = params.name or params.app_mode.value
+            await update(f"App {params.app_mode.value}: {label}…")
+        elif params.action == ComputerAction.scroll:
+            await update(f"Scrolling {params.direction.value} {params.wheel_times}×…")
+        elif params.action == ComputerAction.move:
+            coord = f"({loc[0]}, {loc[1]})" if loc else ""
+            await update(f"Moving pointer to {coord}…")
+        elif params.action == ComputerAction.drag:
+            coord = f"({loc[0]}, {loc[1]})" if loc else ""
+            await update(f"Dragging to {coord}…")
+        elif params.action == ComputerAction.shortcut:
+            await update(f"Shortcut {params.shortcut}…")
+
+    def _run_action(self, desktop: Desktop, params: ComputerSchema) -> str:
+        loc = self._loc(params)
+        if params.action == ComputerAction.click:
+            assert loc is not None
+            desktop.click(loc, button=params.button.value, clicks=params.clicks)
+            return f"Clicked {params.button.value} at {loc[0]},{loc[1]}."
+        if params.action == ComputerAction.type:
+            assert loc is not None
+            desktop.type(
+                loc,
+                text=params.text or "",
+                caret_position=params.caret_position.value,
+                clear=params.clear,
+                press_enter=params.press_enter,
+            )
+            return "Typed text."
+        if params.action == ComputerAction.wait:
+            desktop.wait(params.duration)
+            return f"Waited {params.duration:g} seconds."
+        if params.action == ComputerAction.app:
+            result = desktop.app(mode=params.app_mode.value, name=params.name, loc=loc, size=params.size)
+            return str(result or "App action completed.")
+        if params.action == ComputerAction.scroll:
+            desktop.scroll(
+                loc=loc,
+                orientation=params.orientation.value,
+                direction=params.direction.value,
+                wheel_times=params.wheel_times,
+            )
+            return "Scrolled."
+        if params.action == ComputerAction.move:
+            assert loc is not None
+            desktop.move(loc)
+            return f"Moved pointer to {loc[0]},{loc[1]}."
+        if params.action == ComputerAction.drag:
+            assert loc is not None
+            desktop.drag(loc)
+            return f"Dragged pointer to {loc[0]},{loc[1]}."
+        if params.action == ComputerAction.shortcut:
+            desktop.shortcut(params.shortcut or "")
+            return f"Pressed shortcut {params.shortcut}."
+        return f"Unknown action: {params.action.value}"
+
+    def _loc(self, params: ComputerSchema) -> tuple[int, int] | None:
+        if params.loc is not None:
+            return params.loc
+        if params.x is not None and params.y is not None:
+            return (params.x, params.y)
+        return None