llmcode-cli 1.0.0__py3-none-any.whl

llm_code/tools/web_fetch.py

@@ -0,0 +1,223 @@
+"""WebFetch tool — fetch and extract content from URLs."""
+from __future__ import annotations
+
+from typing import Any, Literal
+
+import httpx
+from pydantic import BaseModel, field_validator
+
+from llm_code.tools.base import PermissionLevel, Tool, ToolResult
+from llm_code.tools.web_common import UrlCache, classify_url, extract_content
+
+
+class WebFetchInput(BaseModel):
+    """Input model for WebFetchTool."""
+
+    url: str
+    prompt: str = ""
+    max_length: int = 50_000
+    raw: bool = False
+    renderer: Literal["auto", "default", "browser"] = "auto"
+
+    @field_validator("url")
+    @classmethod
+    def url_must_not_be_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("url must not be empty")
+        return v.strip()
+
+
+# Module-level cache shared across tool instances
+_cache = UrlCache(max_entries=50, ttl=900.0)
+
+
+class WebFetchTool(Tool):
+    """Tool that fetches and extracts content from a URL."""
+
+    def __init__(self, cache: UrlCache | None = None) -> None:
+        self._cache = cache if cache is not None else _cache
+
+    @property
+    def name(self) -> str:
+        return "web_fetch"
+
+    @property
+    def description(self) -> str:
+        return (
+            "Fetch content from a URL and return it in a readable format. "
+            "Supports HTML (converted to markdown), JSON (pretty-printed), "
+            "and plain text. Results are cached for 15 minutes."
+        )
+
+    @property
+    def required_permission(self) -> PermissionLevel:
+        return PermissionLevel.FULL_ACCESS
+
+    @property
+    def input_schema(self) -> dict:
+        return {
+            "type": "object",
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "description": "The URL to fetch.",
+                },
+                "prompt": {
+                    "type": "string",
+                    "description": "Optional description of what to extract from the page.",
+                    "default": "",
+                },
+                "max_length": {
+                    "type": "integer",
+                    "description": "Maximum content length in characters.",
+                    "default": 50_000,
+                },
+                "raw": {
+                    "type": "boolean",
+                    "description": "If true, skip readability processing for HTML.",
+                    "default": False,
+                },
+                "renderer": {
+                    "type": "string",
+                    "enum": ["auto", "default", "browser"],
+                    "description": (
+                        "Renderer to use: 'auto' (detect playwright availability), "
+                        "'default' (httpx), 'browser' (playwright)."
+                    ),
+                    "default": "auto",
+                },
+            },
+            "required": ["url"],
+        }
+
+    @property
+    def input_model(self) -> type[BaseModel]:
+        return WebFetchInput
+
+    def is_read_only(self, args: dict) -> bool:  # noqa: ARG002
+        return False
+
+    def is_concurrency_safe(self, args: dict) -> bool:  # noqa: ARG002
+        return True
+
+    def _resolve_renderer(self, renderer: str) -> str:
+        """Resolve 'auto' to a concrete renderer based on availability."""
+        if renderer == "auto":
+            try:
+                import playwright  # noqa: F401
+                return "browser"
+            except ImportError:
+                return "default"
+        return renderer
+
+    def _fetch_with_httpx(self, url: str) -> tuple[str, str, int]:
+        """Fetch URL with httpx. Returns (body, content_type, status_code)."""
+        response = httpx.get(url, follow_redirects=True, timeout=30.0)
+        response.raise_for_status()
+        content_type = response.headers.get("content-type", "text/plain")
+        return response.text, content_type, response.status_code
+
+    def _fetch_with_browser(self, url: str) -> tuple[str, str, int]:
+        """Fetch URL with playwright. Falls back to httpx if playwright fails."""
+        try:
+            from playwright.sync_api import sync_playwright
+
+            with sync_playwright() as p:
+                browser = p.chromium.launch(headless=True)
+                try:
+                    page = browser.new_page()
+                    response = page.goto(url, timeout=30_000)
+                    status_code = response.status if response else 200
+                    content = page.content()
+                    return content, "text/html", status_code
+                finally:
+                    browser.close()
+        except Exception:
+            # Fall back to httpx
+            return self._fetch_with_httpx(url)
+
+    def execute(self, args: dict) -> ToolResult:
+        """Execute the web fetch tool."""
+        # Validate and parse input
+        try:
+            from pydantic import ValidationError
+            parsed = WebFetchInput(**args)
+        except (KeyError, TypeError, Exception) as exc:
+            return ToolResult(
+                output=f"Invalid input: {exc}",
+                is_error=True,
+            )
+
+        url = parsed.url
+        max_length = parsed.max_length
+        raw = parsed.raw
+        renderer = parsed.renderer
+
+        # Check URL safety
+        safety = classify_url(url)
+        if safety.is_blocked:
+            reasons = ", ".join(safety.reasons) if safety.reasons else "unknown"
+            return ToolResult(
+                output=f"URL blocked: {url} (reasons: {reasons})",
+                is_error=True,
+                metadata={"url": url, "blocked": True, "reasons": list(safety.reasons)},
+            )
+
+        # Check cache
+        cached_content = self._cache.get(url)
+        if cached_content is not None:
+            return ToolResult(
+                output=cached_content,
+                is_error=False,
+                metadata={
+                    "url": url,
+                    "cached": True,
+                    "status_code": None,
+                    "content_type": None,
+                },
+            )
+
+        # Resolve renderer
+        resolved_renderer = self._resolve_renderer(renderer)
+
+        # Fetch content
+        try:
+            if resolved_renderer == "browser":
+                body, content_type, status_code = self._fetch_with_browser(url)
+            else:
+                body, content_type, status_code = self._fetch_with_httpx(url)
+        except httpx.HTTPStatusError as exc:
+            return ToolResult(
+                output=f"HTTP error {exc.response.status_code}: {exc}",
+                is_error=True,
+                metadata={"url": url, "status_code": exc.response.status_code, "cached": False},
+            )
+        except httpx.RequestError as exc:
+            return ToolResult(
+                output=f"Network error fetching {url}: {exc}",
+                is_error=True,
+                metadata={"url": url, "cached": False},
+            )
+        except Exception as exc:
+            return ToolResult(
+                output=f"Error fetching {url}: {exc}",
+                is_error=True,
+                metadata={"url": url, "cached": False},
+            )
+
+        # Extract content
+        content = extract_content(body, content_type, raw=raw, max_length=max_length)
+
+        # Cache result
+        self._cache.put(url, content)
+
+        return ToolResult(
+            output=content,
+            is_error=False,
+            metadata={
+                "url": url,
+                "status_code": status_code,
+                "content_type": content_type,
+                "cached": False,
+            },
+        )

llm_code/tools/web_search.py

@@ -0,0 +1,280 @@
+"""WebSearchTool — web search using configurable backends."""
+from __future__ import annotations
+
+import fnmatch
+import os
+from urllib.parse import urlparse
+
+from pydantic import BaseModel
+
+from llm_code.tools.base import PermissionLevel, Tool, ToolResult
+from llm_code.tools.search_backends import SearchResult, create_backend
+
+_VALID_BACKENDS = ("auto", "duckduckgo", "brave", "tavily", "searxng")
+
+
+class WebSearchInput(BaseModel):
+    query: str
+    max_results: int = 10
+    backend: str = "auto"
+
+
+class WebSearchTool(Tool):
+    """Tool for performing web searches via configurable backends."""
+
+    @property
+    def name(self) -> str:
+        return "web_search"
+
+    @property
+    def description(self) -> str:
+        return (
+            "Search the web for information. "
+            "Supports DuckDuckGo (default), Brave, Tavily, and SearXNG backends. "
+            "Returns ranked results with titles, URLs, and snippets."
+        )
+
+    @property
+    def input_schema(self) -> dict:
+        return {
+            "type": "object",
+            "properties": {
+                "query": {
+                    "type": "string",
+                    "description": "The search query.",
+                },
+                "max_results": {
+                    "type": "integer",
+                    "description": "Maximum number of results to return (default: 10).",
+                    "default": 10,
+                },
+                "backend": {
+                    "type": "string",
+                    "enum": list(_VALID_BACKENDS),
+                    "description": (
+                        "Search backend to use. 'auto' selects based on config "
+                        "(default: duckduckgo)."
+                    ),
+                    "default": "auto",
+                },
+            },
+            "required": ["query"],
+        }
+
+    @property
+    def required_permission(self) -> PermissionLevel:
+        return PermissionLevel.FULL_ACCESS
+
+    @property
+    def input_model(self) -> type[WebSearchInput]:
+        return WebSearchInput
+
+    def is_read_only(self, args: dict) -> bool:
+        return False
+
+    def is_concurrency_safe(self, args: dict) -> bool:
+        return True
+
+    def _get_web_search_config(self) -> object | None:
+        """Attempt to load WebSearchConfig from runtime config."""
+        try:
+            from llm_code.runtime.config import WebSearchConfig
+            return WebSearchConfig()
+        except ImportError:
+            return None
+
+    def _resolve_backend(self, backend_arg: str) -> tuple[object, str]:
+        """Resolve backend name and instantiate it.
+
+        Returns (backend_instance, backend_name).
+        """
+        cfg = self._get_web_search_config()
+
+        if backend_arg == "auto":
+            backend_name = "duckduckgo"
+            if cfg is not None:
+                backend_name = getattr(cfg, "default_backend", "duckduckgo")
+        else:
+            backend_name = backend_arg
+
+        # Build kwargs for backends that need configuration
+        kwargs: dict = {}
+        if backend_name == "brave" and cfg is not None:
+            api_key_env = getattr(cfg, "brave_api_key_env", "BRAVE_API_KEY")
+            api_key = os.environ.get(api_key_env, "")
+            kwargs["api_key"] = api_key
+        elif backend_name == "tavily" and cfg is not None:
+            api_key_env = getattr(cfg, "tavily_api_key_env", "TAVILY_API_KEY")
+            api_key = os.environ.get(api_key_env, "")
+            kwargs["api_key"] = api_key
+        elif backend_name == "searxng" and cfg is not None:
+            kwargs["base_url"] = getattr(cfg, "searxng_base_url", "")
+
+        backend = create_backend(backend_name, **kwargs)
+        return backend, backend_name
+
+    def _filter_results(
+        self,
+        results: tuple[SearchResult, ...],
+        *,
+        domain_allowlist: tuple[str, ...],
+        domain_denylist: tuple[str, ...],
+    ) -> tuple[SearchResult, ...]:
+        """Apply domain denylist then allowlist filtering.
+
+        Denylist is applied first. If allowlist is non-empty, only results
+        matching an allowlist pattern are kept. Empty lists pass everything.
+
+        Args:
+            results: Results to filter.
+            domain_allowlist: Glob patterns for allowed domains.
+            domain_denylist: Glob patterns for denied domains.
+
+        Returns:
+            Filtered tuple of SearchResult.
+        """
+        def _get_domain(url: str) -> str:
+            try:
+                return urlparse(url).netloc
+            except Exception:
+                return url
+
+        filtered: list[SearchResult] = []
+        for result in results:
+            domain = _get_domain(result.url)
+
+            # Apply denylist first
+            if domain_denylist and any(
+                fnmatch.fnmatch(domain, pattern) for pattern in domain_denylist
+            ):
+                continue
+
+            # Apply allowlist
+            if domain_allowlist and not any(
+                fnmatch.fnmatch(domain, pattern) for pattern in domain_allowlist
+            ):
+                continue
+
+            filtered.append(result)
+
+        return tuple(filtered)
+
+    def _format_results(self, query: str, results: tuple[SearchResult, ...]) -> str:
+        """Format search results as markdown.
+
+        Args:
+            query: The original search query.
+            results: Search results to format.
+
+        Returns:
+            Formatted markdown string.
+        """
+        lines: list[str] = [f'## Search Results for "{query}"', ""]
+
+        if not results:
+            lines.append("(0 results)")
+            return "\n".join(lines)
+
+        for i, result in enumerate(results, start=1):
+            lines.append(f"{i}. **[{result.title}]({result.url})**")
+            lines.append(f"   {result.snippet}")
+            lines.append("")
+
+        lines.append(f"({len(results)} results)")
+        return "\n".join(lines)
+
+    def execute(self, args: dict) -> ToolResult:
+        """Execute a web search.
+
+        Args:
+            args: Dictionary with keys: query (required), max_results (int),
+                  backend (str enum).
+
+        Returns:
+            ToolResult with formatted search results, or error.
+        """
+        query = args.get("query", "")
+        if not query or not str(query).strip():
+            return ToolResult(
+                output="Error: 'query' is required and must not be empty.",
+                is_error=True,
+            )
+
+        max_results = int(args.get("max_results", 10))
+        backend_arg = str(args.get("backend", "auto"))
+
+        # Apply domain filtering from config
+        cfg = self._get_web_search_config()
+        allowlist: tuple[str, ...] = ()
+        denylist: tuple[str, ...] = ()
+        if cfg is not None:
+            allowlist = getattr(cfg, "domain_allowlist", ())
+            denylist = getattr(cfg, "domain_denylist", ())
+
+        if backend_arg == "auto":
+            # Fallback chain: try each configured backend until one returns results
+            results = self._search_with_fallback(query, max_results, cfg)
+        else:
+            try:
+                backend, _name = self._resolve_backend(backend_arg)
+            except (ValueError, Exception) as exc:
+                return ToolResult(
+                    output=f"Error: Failed to initialize search backend: {exc}",
+                    is_error=True,
+                )
+            try:
+                results = backend.search(query, max_results=max_results)
+            except Exception as exc:
+                return ToolResult(
+                    output=f"Error: Search failed: {exc}",
+                    is_error=True,
+                )
+
+        results = self._filter_results(results, domain_allowlist=allowlist, domain_denylist=denylist)
+        output = self._format_results(query, results)
+        return ToolResult(output=output, is_error=False)
+
+    def _search_with_fallback(
+        self, query: str, max_results: int, cfg: object | None,
+    ) -> tuple[SearchResult, ...]:
+        """Try backends in order until one returns results.
+
+        Fallback order: duckduckgo -> brave -> searxng -> tavily.
+        Only backends that are configured (have API keys / base_url set) are tried.
+        """
+        # Build ordered list of (backend_name, kwargs) to try
+        chain: list[tuple[str, dict]] = []
+
+        # 1. DuckDuckGo (always available, no config needed)
+        chain.append(("duckduckgo", {}))
+
+        # 2. Brave (if API key configured)
+        if cfg is not None:
+            brave_key_env = getattr(cfg, "brave_api_key_env", "BRAVE_API_KEY")
+            brave_key = os.environ.get(brave_key_env, "")
+            if brave_key:
+                chain.append(("brave", {"api_key": brave_key}))
+
+        # 3. SearXNG (if base_url configured)
+        if cfg is not None:
+            searxng_url = getattr(cfg, "searxng_base_url", "")
+            if searxng_url:
+                chain.append(("searxng", {"base_url": searxng_url}))
+
+        # 4. Tavily (if API key configured)
+        if cfg is not None:
+            tavily_key_env = getattr(cfg, "tavily_api_key_env", "TAVILY_API_KEY")
+            tavily_key = os.environ.get(tavily_key_env, "")
+            if tavily_key:
+                chain.append(("tavily", {"api_key": tavily_key}))
+
+        for backend_name, kwargs in chain:
+            try:
+                backend = create_backend(backend_name, **kwargs)
+                results = backend.search(query, max_results=max_results)
+                if results:
+                    return results
+            except Exception:
+                continue
+
+        return ()

llm_code/tools/write_file.py

@@ -0,0 +1,118 @@
+"""WriteFileTool — writes content to a file, auto-creating parent directories."""
+from __future__ import annotations
+
+import pathlib
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel
+
+from llm_code.runtime.file_protection import check_write
+from llm_code.tools.base import PermissionLevel, Tool, ToolResult
+from llm_code.utils.errors import friendly_error
+
+if TYPE_CHECKING:
+    from llm_code.runtime.overlay import OverlayFS
+
+
+class WriteFileInput(BaseModel):
+    path: str
+    content: str
+
+
+class WriteFileTool(Tool):
+    @property
+    def name(self) -> str:
+        return "write_file"
+
+    @property
+    def description(self) -> str:
+        return "Write content to a file, creating parent directories as needed."
+
+    @property
+    def input_schema(self) -> dict:
+        return {
+            "type": "object",
+            "properties": {
+                "path": {"type": "string", "description": "Absolute path to write to"},
+                "content": {"type": "string", "description": "Content to write"},
+            },
+            "required": ["path", "content"],
+        }
+
+    @property
+    def required_permission(self) -> PermissionLevel:
+        return PermissionLevel.WORKSPACE_WRITE
+
+    @property
+    def input_model(self) -> type[WriteFileInput]:
+        return WriteFileInput
+
+    def execute(self, args: dict, overlay: "OverlayFS | None" = None) -> ToolResult:
+        path = pathlib.Path(args["path"])
+        content: str = args["content"]
+
+        protection = check_write(str(path))
+        if not protection.allowed:
+            return ToolResult(output=protection.reason, is_error=True)
+        if protection.severity == "warn":
+            # Surface the warning in output metadata; execution still proceeds
+            warning_prefix = f"[WARNING] {protection.reason}\n"
+        else:
+            warning_prefix = ""
+
+        if overlay is not None:
+            # Speculative mode: write to overlay, read old content from overlay/real FS
+            old_content: str | None = None
+            try:
+                old_content = overlay.read(path)
+            except FileNotFoundError:
+                pass
+
+            overlay.write(path, content)
+
+            line_count = len(content.splitlines())
+            output = warning_prefix + f"Wrote {line_count} lines to {path}"
+
+            metadata: dict | None = None
+            if old_content is not None and old_content != content:
+                from llm_code.utils.diff import generate_diff, count_changes
+
+                hunks = generate_diff(old_content, content, path.name)
+                adds, dels = count_changes(hunks)
+                metadata = {
+                    "diff": [h.to_dict() for h in hunks],
+                    "additions": adds,
+                    "deletions": dels,
+                }
+
+            return ToolResult(output=output, metadata=metadata)
+
+        # Normal mode: write directly to the real filesystem
+        # Capture old content if overwriting
+        old_content = None
+        if path.exists():
+            old_content = path.read_text()
+
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(content)
+        except (PermissionError, OSError) as exc:
+            return ToolResult(output=friendly_error(exc, str(path)), is_error=True)
+
+        line_count = len(content.splitlines())
+        output = warning_prefix + f"Wrote {line_count} lines to {path}"
+
+        # Generate diff for overwrites
+        metadata = None
+        if old_content is not None and old_content != content:
+            from llm_code.utils.diff import generate_diff, count_changes
+
+            hunks = generate_diff(old_content, content, path.name)
+            adds, dels = count_changes(hunks)
+            metadata = {
+                "diff": [h.to_dict() for h in hunks],
+                "additions": adds,
+                "deletions": dels,
+            }
+
+        return ToolResult(output=output, metadata=metadata)

llm_code/tui/__init__.py

@@ -0,0 +1 @@
+"""Fullscreen TUI package using Textual."""