tunacode-cli 0.1.21__py3-none-any.whl

tunacode/tools/web_fetch.py

@@ -0,0 +1,245 @@
+"""
+Web fetch tool for TunaCode - HTTP GET requests with HTML-to-text conversion.
+
+This tool provides web content fetching with:
+- HTTP GET requests with configurable timeout
+- HTML-to-text conversion for readable output
+- URL security validation (blocks localhost, private IPs, file://)
+- Content size limiting (5MB max)
+
+CLAUDE_ANCHOR[web-fetch-module]: HTTP GET with HTML-to-text conversion
+"""
+
+import ipaddress
+import re
+from urllib.parse import urlparse
+
+import html2text
+import httpx
+from pydantic_ai.exceptions import ModelRetry
+
+from tunacode.tools.decorators import base_tool
+
+# Constants
+MAX_CONTENT_SIZE = 5 * 1024 * 1024  # 5MB
+MAX_OUTPUT_SIZE = 100 * 1024  # 100KB for output truncation
+DEFAULT_TIMEOUT = 60  # seconds
+USER_AGENT = "TunaCode/1.0 (https://tunacode.xyz)"
+
+# Private IP ranges to block
+PRIVATE_IP_PATTERNS = [
+    re.compile(r"^127\."),  # 127.x.x.x
+    re.compile(r"^10\."),  # 10.x.x.x
+    re.compile(r"^172\.(1[6-9]|2[0-9]|3[01])\."),  # 172.16-31.x.x
+    re.compile(r"^192\.168\."),  # 192.168.x.x
+    re.compile(r"^0\."),  # 0.x.x.x
+    re.compile(r"^169\.254\."),  # Link-local
+    re.compile(r"^::1$"),  # IPv6 localhost
+    re.compile(r"^fe80:"),  # IPv6 link-local
+    re.compile(r"^fc00:"),  # IPv6 unique local
+    re.compile(r"^fd00:"),  # IPv6 unique local
+]
+
+# Blocked hostnames
+BLOCKED_HOSTNAMES = frozenset(
+    [
+        "localhost",
+        "localhost.localdomain",
+        "local",
+        "0.0.0.0",  # nosec B104 - this is a blocklist, not a bind address
+        "127.0.0.1",
+        "::1",
+    ]
+)
+
+
+def _is_private_ip(ip_str: str) -> bool:
+    """Check if an IP address is private/reserved."""
+    for pattern in PRIVATE_IP_PATTERNS:
+        if pattern.match(ip_str):
+            return True
+
+    try:
+        ip = ipaddress.ip_address(ip_str)
+        return ip.is_private or ip.is_loopback or ip.is_reserved or ip.is_link_local
+    except ValueError:
+        return False
+
+
+def _validate_url(url: str) -> str:
+    """Validate URL for security.
+
+    Args:
+        url: URL to validate
+
+    Returns:
+        Validated URL
+
+    Raises:
+        ModelRetry: If URL is invalid or blocked
+    """
+    if not url or not url.strip():
+        raise ModelRetry("URL cannot be empty.")
+
+    url = url.strip()
+
+    try:
+        parsed = urlparse(url)
+    except Exception as err:
+        raise ModelRetry(f"Invalid URL format: {url}") from err
+
+    # Check scheme
+    if parsed.scheme not in ("http", "https"):
+        raise ModelRetry(
+            f"Invalid URL scheme '{parsed.scheme}'. Only http:// and https:// are allowed."
+        )
+
+    # Check hostname presence
+    if not parsed.hostname:
+        raise ModelRetry(f"URL missing hostname: {url}")
+
+    hostname = parsed.hostname.lower()
+
+    # Block known localhost hostnames
+    if hostname in BLOCKED_HOSTNAMES:
+        raise ModelRetry(f"Blocked URL: {url}. Cannot fetch from localhost or local addresses.")
+
+    # Check if hostname is an IP address and validate
+    if _is_private_ip(hostname):
+        raise ModelRetry(f"Blocked URL: {url}. Cannot fetch from private or reserved IP addresses.")
+
+    return url
+
+
+def _convert_html_to_text(html_content: str) -> str:
+    """Convert HTML to readable plain text.
+
+    Args:
+        html_content: Raw HTML content
+
+    Returns:
+        Plain text extracted from HTML
+    """
+    converter = html2text.HTML2Text()
+    converter.ignore_links = False
+    converter.ignore_images = True
+    converter.ignore_emphasis = False
+    converter.body_width = 80
+    converter.unicode_snob = True
+    converter.skip_internal_links = True
+
+    return converter.handle(html_content)
+
+
+def _truncate_output(content: str, max_size: int = MAX_OUTPUT_SIZE) -> str:
+    """Truncate content if it exceeds max size.
+
+    Args:
+        content: Content to truncate
+        max_size: Maximum size in bytes
+
+    Returns:
+        Truncated content with indicator if truncated
+    """
+    if len(content.encode("utf-8")) <= max_size:
+        return content
+
+    # Truncate to approximate character count
+    truncated = content[: max_size // 2]
+    return truncated + "\n\n... [Content truncated due to size] ..."
+
+
+@base_tool
+async def web_fetch(
+    url: str,
+    timeout: int = DEFAULT_TIMEOUT,
+) -> str:
+    """Fetch web content from a URL and return as readable text.
+
+    Args:
+        url: The URL to fetch (http:// or https://)
+        timeout: Request timeout in seconds (default: 60)
+
+    Returns:
+        Readable text content from the URL
+    """
+    # Validate URL security
+    validated_url = _validate_url(url)
+
+    # Clamp timeout to reasonable bounds
+    timeout = max(5, min(timeout, 120))
+
+    try:
+        async with httpx.AsyncClient(
+            timeout=httpx.Timeout(timeout),
+            follow_redirects=True,
+            max_redirects=5,
+            headers={"User-Agent": USER_AGENT},
+        ) as client:
+            # First, do a HEAD request to check content size
+            try:
+                head_response = await client.head(validated_url)
+                content_length = head_response.headers.get("content-length")
+                if content_length and int(content_length) > MAX_CONTENT_SIZE:
+                    raise ModelRetry(
+                        f"Content too large ({int(content_length) // 1024 // 1024}MB). "
+                        f"Maximum allowed is {MAX_CONTENT_SIZE // 1024 // 1024}MB."
+                    )
+            except httpx.HTTPError:
+                # HEAD failed, proceed with GET and stream check
+                pass
+
+            # Fetch the actual content
+            response = await client.get(validated_url)
+            response.raise_for_status()
+
+            # Check final URL after redirects for security
+            final_url = str(response.url)
+            if final_url != validated_url:
+                _validate_url(final_url)
+
+            # Check content size
+            content = response.content
+            if len(content) > MAX_CONTENT_SIZE:
+                raise ModelRetry(
+                    f"Content too large ({len(content) // 1024 // 1024}MB). "
+                    f"Maximum allowed is {MAX_CONTENT_SIZE // 1024 // 1024}MB."
+                )
+
+            # Decode content
+            try:
+                text_content = content.decode("utf-8")
+            except UnicodeDecodeError:
+                text_content = content.decode("latin-1", errors="replace")
+
+            # Convert HTML to text if content is HTML
+            content_type = response.headers.get("content-type", "").lower()
+            if "text/html" in content_type or "<html" in text_content[:1000].lower():
+                text_content = _convert_html_to_text(text_content)
+
+            # Truncate if too large
+            text_content = _truncate_output(text_content)
+
+            return text_content
+
+    except httpx.TimeoutException as err:
+        msg = f"Request timed out after {timeout} seconds. Try again or use a shorter timeout."
+        raise ModelRetry(msg) from err
+    except httpx.TooManyRedirects as err:
+        msg = f"Too many redirects while fetching {url}. The URL may be invalid."
+        raise ModelRetry(msg) from err
+    except httpx.HTTPStatusError as err:
+        status = err.response.status_code
+        if status == 404:
+            raise ModelRetry(f"Page not found (404): {url}. Check the URL.") from err
+        if status == 403:
+            msg = f"Access forbidden (403): {url}. The page may require authentication."
+            raise ModelRetry(msg) from err
+        if status == 429:
+            raise ModelRetry(f"Rate limited (429): {url}. Try again later.") from err
+        if status >= 500:
+            msg = f"Server error ({status}): {url}. The server may be down."
+            raise ModelRetry(msg) from err
+        raise ModelRetry(f"HTTP error {status} fetching {url}") from err
+    except httpx.RequestError as err:
+        raise ModelRetry(f"Failed to connect to {url}: {err}") from err

tunacode/tools/write_file.py

@@ -0,0 +1,34 @@
+"""File writing tool for agent operations."""
+
+import os
+
+from pydantic_ai.exceptions import ModelRetry
+
+from tunacode.tools.decorators import file_tool
+
+
+@file_tool(writes=True)
+async def write_file(filepath: str, content: str) -> str:
+    """Write content to a new file. Fails if the file already exists.
+
+    Args:
+        filepath: The absolute path to the file to write.
+        content: The content to write to the file.
+
+    Returns:
+        A message indicating success.
+    """
+    if os.path.exists(filepath):
+        raise ModelRetry(
+            f"File '{filepath}' already exists. "
+            "Use the `update_file` tool to modify it, or choose a different filepath."
+        )
+
+    dirpath = os.path.dirname(filepath)
+    if dirpath and not os.path.exists(dirpath):
+        os.makedirs(dirpath, exist_ok=True)
+
+    with open(filepath, "w", encoding="utf-8") as f:
+        f.write(content)
+
+    return f"Successfully wrote to new file: {filepath}"

tunacode/tools/xml_helper.py

@@ -0,0 +1,34 @@
+"""Helper module for loading prompts and schemas from XML files."""
+
+from functools import lru_cache
+from pathlib import Path
+
+from defusedxml.ElementTree import ParseError
+from defusedxml.ElementTree import parse as xml_parse
+
+
+@lru_cache(maxsize=32)
+def load_prompt_from_xml(tool_name: str) -> str | None:
+    """Load and return the base prompt from XML file.
+
+    Args:
+        tool_name: Name of the tool (e.g., 'grep', 'glob')
+
+    Returns:
+        str: The loaded prompt from XML or None if not found
+    """
+    prompt_file = Path(__file__).parent / "prompts" / f"{tool_name}_prompt.xml"
+    if not prompt_file.exists():
+        return None
+
+    try:
+        tree = xml_parse(prompt_file)
+    except ParseError:
+        return None
+
+    root = tree.getroot()
+    description = root.find("description")
+    if description is None or description.text is None:
+        return None
+
+    return description.text.strip()

tunacode/types/__init__.py

@@ -0,0 +1,166 @@
+"""Centralized type definitions for TunaCode CLI.
+
+This package contains all type aliases, protocols, and type definitions
+used throughout the TunaCode codebase.
+
+All types are re-exported from this module for backward compatibility.
+"""
+
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+# Base types
+from tunacode.types.base import (
+    AgentConfig,
+    AgentName,
+    CommandArgs,
+    CommandResult,
+    ConfigFile,
+    ConfigPath,
+    CostAmount,
+    DeviceId,
+    DiffHunk,
+    DiffLine,
+    EnvConfig,
+    ErrorContext,
+    ErrorMessage,
+    FileContent,
+    FileDiff,
+    FileEncoding,
+    FilePath,
+    FileSize,
+    InputSessions,
+    LineNumber,
+    ModelName,
+    OriginalError,
+    SessionId,
+    TokenCount,
+    ToolArgs,
+    ToolCallId,
+    ToolName,
+    ToolResult,
+    UpdateOperation,
+    UserConfig,
+    ValidationResult,
+    Validator,
+)
+
+# Callback types
+from tunacode.types.callbacks import (
+    AsyncFunc,
+    AsyncToolFunc,
+    AsyncVoidFunc,
+    ToolCallback,
+    ToolProgress,
+    ToolProgressCallback,
+    ToolStartCallback,
+    UICallback,
+    UIInputCallback,
+)
+
+# Dataclasses
+from tunacode.types.dataclasses import (
+    AgentState,
+    CommandContext,
+    CostBreakdown,
+    FallbackResponse,
+    ModelConfig,
+    ModelPricing,
+    ModelRegistry,
+    ResponseState,
+    TokenUsage,
+    ToolConfirmationRequest,
+    ToolConfirmationResponse,
+)
+
+# Pydantic-AI wrappers
+from tunacode.types.pydantic_ai import (
+    AgentResponse,
+    AgentRun,
+    MessageHistory,
+    MessagePart,
+    ModelRequest,
+    ModelResponse,
+    PydanticAgent,
+)
+
+# State protocol
+from tunacode.types.state import (
+    SessionStateProtocol,
+    StateManager,
+    StateManagerProtocol,
+)
+
+# Backward compatibility: ProcessRequestCallback needs StateManager
+ProcessRequestCallback = Callable[[str, StateManager, bool], Awaitable[Any]]
+
+__all__ = [
+    # Base types
+    "AgentConfig",
+    "AgentName",
+    "CommandArgs",
+    "CommandResult",
+    "ConfigFile",
+    "ConfigPath",
+    "CostAmount",
+    "DeviceId",
+    "DiffHunk",
+    "DiffLine",
+    "EnvConfig",
+    "ErrorContext",
+    "ErrorMessage",
+    "FileContent",
+    "FileDiff",
+    "FileEncoding",
+    "FilePath",
+    "FileSize",
+    "InputSessions",
+    "LineNumber",
+    "ModelName",
+    "OriginalError",
+    "SessionId",
+    "TokenCount",
+    "ToolArgs",
+    "ToolCallId",
+    "ToolName",
+    "ToolResult",
+    "UpdateOperation",
+    "UserConfig",
+    "ValidationResult",
+    "Validator",
+    # Pydantic-AI
+    "AgentResponse",
+    "AgentRun",
+    "MessageHistory",
+    "MessagePart",
+    "ModelRequest",
+    "ModelResponse",
+    "PydanticAgent",
+    # Callbacks
+    "AsyncFunc",
+    "AsyncToolFunc",
+    "AsyncVoidFunc",
+    "ProcessRequestCallback",
+    "ToolCallback",
+    "ToolProgress",
+    "ToolProgressCallback",
+    "ToolStartCallback",
+    "UICallback",
+    "UIInputCallback",
+    # State
+    "SessionStateProtocol",
+    "StateManager",
+    "StateManagerProtocol",
+    # Dataclasses
+    "AgentState",
+    "CommandContext",
+    "CostBreakdown",
+    "FallbackResponse",
+    "ModelConfig",
+    "ModelPricing",
+    "ModelRegistry",
+    "ResponseState",
+    "TokenUsage",
+    "ToolConfirmationRequest",
+    "ToolConfirmationResponse",
+]

tunacode/types/base.py

@@ -0,0 +1,94 @@
+"""Base type aliases for TunaCode CLI.
+
+Contains fundamental type definitions that have no external dependencies
+beyond Python stdlib.
+"""
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+# Identity types - string wrappers for semantic clarity
+ModelName = str
+ToolName = str
+SessionId = str
+DeviceId = str
+AgentName = str
+ToolCallId = str
+
+# File system types
+FilePath = str | Path
+FileContent = str
+FileEncoding = str
+FileDiff = tuple[str, str]
+FileSize = int
+LineNumber = int
+ConfigPath = Path
+ConfigFile = Path
+
+# Configuration types
+UserConfig = dict[str, Any]
+EnvConfig = dict[str, str]
+InputSessions = dict[str, Any]
+AgentConfig = dict[str, Any]
+
+# Tool types
+ToolArgs = dict[str, Any]
+ToolResult = str
+
+# Error handling types
+ErrorContext = dict[str, Any]
+OriginalError = Exception | None
+ErrorMessage = str
+
+# Diff types
+UpdateOperation = dict[str, Any]
+DiffLine = str
+DiffHunk = list[DiffLine]
+
+# Validation types
+ValidationResult = bool | str
+Validator = Callable[[Any], ValidationResult]
+
+# Token/Cost types
+TokenCount = int
+CostAmount = float
+
+# Command types
+CommandArgs = list[str]
+CommandResult = Any | None
+
+__all__ = [
+    "AgentConfig",
+    "AgentName",
+    "CommandArgs",
+    "CommandResult",
+    "ConfigFile",
+    "ConfigPath",
+    "CostAmount",
+    "DeviceId",
+    "DiffHunk",
+    "DiffLine",
+    "EnvConfig",
+    "ErrorContext",
+    "ErrorMessage",
+    "FileContent",
+    "FileDiff",
+    "FileEncoding",
+    "FilePath",
+    "FileSize",
+    "InputSessions",
+    "LineNumber",
+    "ModelName",
+    "OriginalError",
+    "SessionId",
+    "TokenCount",
+    "ToolArgs",
+    "ToolCallId",
+    "ToolName",
+    "ToolResult",
+    "UpdateOperation",
+    "UserConfig",
+    "ValidationResult",
+    "Validator",
+]

tunacode/types/callbacks.py

@@ -0,0 +1,53 @@
+"""Callback type definitions for TunaCode CLI.
+
+Contains callback signatures and the ToolProgress dataclass for
+structured progress reporting.
+"""
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class ToolProgress:
+    """Structured progress information for subagent tool execution.
+
+    Attributes:
+        subagent: Name of the subagent (e.g., "research")
+        operation: Description of current operation (e.g., "grep pattern...")
+        current: Current operation count (1-indexed)
+        total: Total expected operations (0 if unknown)
+    """
+
+    subagent: str
+    operation: str
+    current: int
+    total: int
+
+
+# Tool callbacks
+ToolCallback = Callable[[Any, Any], Awaitable[None]]
+ToolStartCallback = Callable[[str], None]
+ToolProgressCallback = Callable[[ToolProgress], None]
+
+# UI callbacks
+UICallback = Callable[[str], Awaitable[None]]
+UIInputCallback = Callable[[str, str], Awaitable[str]]
+
+# Async function types
+AsyncFunc = Callable[..., Awaitable[Any]]
+AsyncToolFunc = Callable[..., Awaitable[str]]
+AsyncVoidFunc = Callable[..., Awaitable[None]]
+
+__all__ = [
+    "AsyncFunc",
+    "AsyncToolFunc",
+    "AsyncVoidFunc",
+    "ToolCallback",
+    "ToolProgress",
+    "ToolProgressCallback",
+    "ToolStartCallback",
+    "UICallback",
+    "UIInputCallback",
+]