crabcode 0.1.6__py3-none-win_amd64.whl

crabcode_core/tools/tool_search.py

@@ -0,0 +1,64 @@
+"""Discover and activate tool schemas without executing the discovered tools."""
+
+from crabcode_core.types.tool import PermissionBehavior, PermissionResult, Tool, ToolContext, ToolResult
+
+
+class ToolSearchTool(Tool):
+    name = "ToolSearch"
+    description = (
+        "Find and load tools by exact names, group, or keyword query. Their schemas "
+        "become available in your NEXT response, never in the same batch. "
+        "Use list=true to browse without loading; paginate with offset. "
+        "Loading does not execute tools or grant permissions."
+    )
+    is_read_only = True
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "names": {
+                "type": "array", "items": {"type": "string"},
+                "description": (
+                    "Exact registered tool names from the directory's names field; "
+                    "do not add a group prefix."
+                ),
+            },
+            "group": {
+                "type": "string",
+                "description": "Tool category to search or load. This is not a tool-name namespace.",
+            },
+            "query": {"type": "string"},
+            "list": {"type": "boolean", "default": False},
+            "offset": {"type": "integer", "minimum": 0, "default": 0},
+            "limit": {"type": "integer", "minimum": 1, "maximum": 30, "default": 5},
+        },
+        "additionalProperties": False,
+    }
+
+    async def check_permissions(self, tool_input, context):
+        return PermissionResult(behavior=PermissionBehavior.ALLOW)
+
+    async def validate_input(self, tool_input):
+        names = tool_input.get("names", [])
+        if not isinstance(names, list) or any(not isinstance(name, str) for name in names):
+            return "names must be an array of tool names"
+        if any(not isinstance(tool_input.get(key, ""), str) for key in ("query", "group")):
+            return "query and group must be strings"
+        if not isinstance(tool_input.get("list", False), bool):
+            return "list must be a boolean"
+        offset, limit = tool_input.get("offset", 0), tool_input.get("limit", 5)
+        if type(offset) is not int or offset < 0:
+            return "offset must be a nonnegative integer"
+        if type(limit) is not int or not 1 <= limit <= 30:
+            return "limit must be an integer between 1 and 30"
+        return None
+
+    async def call(self, tool_input: dict, context: ToolContext) -> ToolResult:
+        if context.tool_catalog is None:
+            return ToolResult(result_for_model="Tool discovery is unavailable in this context.", is_error=True)
+        result = context.tool_catalog.search(
+            names=tool_input.get("names", []), group=tool_input.get("group", ""),
+            query=tool_input.get("query", ""),
+            list_only=tool_input.get("list", False) or not any(tool_input.get(k) for k in ("names", "group", "query")),
+            offset=tool_input.get("offset", 0), limit=tool_input.get("limit", 5),
+        )
+        return ToolResult(result_for_model=result)

crabcode_core/tools/web_search.py

@@ -0,0 +1,444 @@
+"""WebSearchTool — search the web using Tavily, DuckDuckGo, or Bing."""
+
+from __future__ import annotations
+
+import os
+import re
+import time
+import xml.etree.ElementTree as ET
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qs, quote, unquote, urljoin, urlparse
+
+import httpx
+
+from crabcode_core.logging_utils import get_logger
+from crabcode_core.types.tool import PermissionBehavior, PermissionResult, Tool, ToolContext, ToolResult
+
+logger = get_logger(__name__)
+
+_DEFAULT_TIMEOUT = 8
+_DEFAULT_MAX_RESULTS = 5
+_MAX_RESULTS = 10
+_DDG_BASE_URL = "https://html.duckduckgo.com/html/"
+_DDG_RESULT_BASE_URL = "https://duckduckgo.com"
+_BING_HOME_URL = "https://www.bing.com/"
+_BING_SEARCH_URL = "https://www.bing.com/search"
+_TAVILY_URL = "https://api.tavily.com/search"
+_BROWSER_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+_DDG_BACKOFF_SECONDS = 10 * 60
+_TAG_RE = re.compile(r"<[^>]+>")
+_ddg_unavailable_until = 0.0
+
+
+class DuckDuckGoUnavailable(RuntimeError):
+    """DuckDuckGo did not return a results page."""
+
+
+def _reset_ddg_backoff() -> None:
+    """Clear the in-process DuckDuckGo backoff. Used by tests."""
+    global _ddg_unavailable_until
+    _ddg_unavailable_until = 0.0
+
+
+def _ddg_in_backoff() -> bool:
+    return time.monotonic() < _ddg_unavailable_until
+
+
+def _note_ddg_unavailable() -> None:
+    global _ddg_unavailable_until
+    _ddg_unavailable_until = time.monotonic() + _DDG_BACKOFF_SECONDS
+
+
+def _is_ddg_challenge(status_code: int, html: str) -> bool:
+    """True when DuckDuckGo served a bot check instead of results.
+
+    HTTP 202 plus an anomaly page still passes ``raise_for_status`` and has no
+    result links, which used to be reported as an empty search.
+    """
+    if status_code == 202:
+        return True
+    lowered = html.lower()
+    return "anomaly-modal" in lowered or "bots use duckduckgo" in lowered
+
+
+def _local_name(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]
+
+
+def _plain(value: str) -> str:
+    return _clean_text(_TAG_RE.sub(" ", value))
+
+
+def _is_http_url(url: str) -> bool:
+    parsed = urlparse(url)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+
+def _resolve_ddg_result_url(url: str) -> str:
+    raw = url.strip()
+    if not raw:
+        return ""
+
+    if raw.startswith("//"):
+        raw = f"https:{raw}"
+    elif raw.startswith("/"):
+        raw = urljoin(_DDG_RESULT_BASE_URL, raw)
+
+    parsed = urlparse(raw)
+    if parsed.netloc.endswith("duckduckgo.com") and parsed.path.startswith("/l/"):
+        uddg = parse_qs(parsed.query).get("uddg", [])
+        if uddg:
+            return unquote(uddg[0]).strip()
+
+    return raw
+
+
+def _clean_text(value: str) -> str:
+    return " ".join(value.split())
+
+
+class _DuckDuckGoHTMLParser(HTMLParser):
+    """Extract result links from DuckDuckGo's HTML results page."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.results: list[dict[str, str]] = []
+        self._current: dict[str, str] | None = None
+        self._capture_title = False
+        self._capture_snippet = False
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attrs_map = dict(attrs)
+        class_name = attrs_map.get("class", "")
+        if tag == "a" and "result__a" in class_name:
+            self._flush_current()
+            href = attrs_map.get("href") or ""
+            self._current = {"title": "", "url": href, "snippet": ""}
+            self._capture_title = True
+            return
+        if self._current and tag in {"a", "div", "span"} and "result__snippet" in class_name:
+            self._capture_snippet = True
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "a" and self._capture_title:
+            self._capture_title = False
+        if tag in {"a", "div", "span"} and self._capture_snippet:
+            self._capture_snippet = False
+        if tag == "article" and self._current:
+            self._flush_current()
+
+    def handle_data(self, data: str) -> None:
+        if not self._current:
+            return
+        if self._capture_title:
+            self._current["title"] += data
+        elif self._capture_snippet:
+            self._current["snippet"] += data
+
+    def close(self) -> None:
+        super().close()
+        self._flush_current()
+
+    def _flush_current(self) -> None:
+        if not self._current:
+            return
+        title = _clean_text(self._current.get("title", ""))
+        url = _resolve_ddg_result_url(self._current.get("url", ""))
+        snippet = _clean_text(self._current.get("snippet", ""))
+        if title and _is_http_url(url):
+            self.results.append({"title": title, "url": url, "snippet": snippet})
+        self._current = None
+        self._capture_title = False
+        self._capture_snippet = False
+
+
+def _parse_ddg_html(html: str) -> list[dict[str, str]]:
+    parser = _DuckDuckGoHTMLParser()
+    parser.feed(html)
+    parser.close()
+    return parser.results
+
+
+def _parse_bing_rss(xml_text: str) -> list[dict[str, str]]:
+    stripped = xml_text.lstrip("\ufeff \t\r\n")
+    lowered = stripped[:200].lower()
+    if lowered.startswith("<!doctype html") or lowered.startswith("<html"):
+        raise RuntimeError("Bing returned an HTML page instead of search results")
+    try:
+        root = ET.fromstring(xml_text)
+    except ET.ParseError as exc:
+        raise RuntimeError("Bing returned a response that was not RSS") from exc
+
+    results: list[dict[str, str]] = []
+    for item in root.iter():
+        if _local_name(item.tag) != "item":
+            continue
+        fields = {_local_name(child.tag): child.text or "" for child in list(item)}
+        results.append(
+            {
+                "title": _plain(fields.get("title", "")),
+                "url": fields.get("link", "").strip(),
+                "snippet": _plain(fields.get("description", "")),
+            }
+        )
+    return results
+
+
+class WebSearchTool(Tool):
+    name = "WebSearch"
+    description = "Search the web for current external information using Tavily, DuckDuckGo, or Bing."
+    is_read_only = True
+    is_concurrency_safe = True
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "Search query to run on the public web.",
+            },
+            "num_results": {
+                "type": "integer",
+                "description": "Maximum number of results to return (default 5, max 10).",
+                "default": _DEFAULT_MAX_RESULTS,
+            },
+        },
+        "required": ["query"],
+    }
+
+    def __init__(self) -> None:
+        self._tool_config: dict[str, Any] = {}
+        self._provider = "auto"
+        self._timeout = _DEFAULT_TIMEOUT
+        self._max_results = _DEFAULT_MAX_RESULTS
+        self._api_key_env = "TAVILY_API_KEY"
+        self._api_key: str | None = None
+
+    async def setup(self, context: ToolContext) -> None:
+        await super().setup(context)
+        self._tool_config = dict(context.tool_config)
+        self._provider = str(self._tool_config.get("provider", "auto")).lower()
+        self._timeout = int(self._tool_config.get("timeout_seconds", _DEFAULT_TIMEOUT))
+        self._max_results = max(1, min(int(self._tool_config.get("max_results", _DEFAULT_MAX_RESULTS)), _MAX_RESULTS))
+        self._api_key_env = str(self._tool_config.get("api_key_env", "TAVILY_API_KEY"))
+        self._api_key = os.environ.get(self._api_key_env) or context.env.get(self._api_key_env)
+
+        if self._provider not in {"auto", "tavily", "ddg"}:
+            logger.warning("Unknown WebSearch provider %r; falling back to auto", self._provider)
+            self._provider = "auto"
+
+        if self._provider == "tavily" and not self._api_key:
+            logger.info("Disabling WebSearch: provider=tavily but %s is not configured", self._api_key_env)
+            self.is_enabled = False
+            return
+
+        if not await self._has_connectivity():
+            logger.info("Disabling WebSearch: outbound network probe failed")
+            self.is_enabled = False
+
+    async def get_prompt(self, **kwargs: Any) -> str:
+        return (
+            "Search the public web for current external information. "
+            "Use this instead of shell-based web access when you need recent facts, docs, or sources outside the repo. "
+            "Returns a compact list of result titles, URLs, and snippets. "
+            "This tool performs search only; it does not fetch and read full pages."
+        )
+
+    async def validate_input(self, tool_input: dict[str, Any]) -> str | None:
+        query = str(tool_input.get("query", "")).strip()
+        if not query:
+            return "query is required"
+        raw_num = tool_input.get("num_results", _DEFAULT_MAX_RESULTS)
+        try:
+            int(raw_num)
+        except (TypeError, ValueError):
+            return "num_results must be an integer"
+        return None
+
+    async def check_permissions(
+        self,
+        tool_input: dict[str, Any],
+        context: ToolContext,
+    ) -> PermissionResult:
+        return PermissionResult(
+            behavior=PermissionBehavior.ASK,
+            reason="WebSearch requires confirmation before making a network request",
+        )
+
+    async def call(
+        self,
+        tool_input: dict[str, Any],
+        context: ToolContext,
+    ) -> ToolResult:
+        if not self.is_enabled:
+            return ToolResult(
+                result_for_model="WebSearch is unavailable in this session because network access was not detected during initialization.",
+                is_error=True,
+            )
+
+        query = str(tool_input.get("query", "")).strip()
+        if not query:
+            return ToolResult(result_for_model="query is required.", is_error=True)
+
+        num_results = max(1, min(int(tool_input.get("num_results", self._max_results)), _MAX_RESULTS))
+
+        try:
+            provider, results = await self._run_search(query, num_results)
+        except DuckDuckGoUnavailable as exc:
+            logger.warning("WebSearch DuckDuckGo unavailable for query %r: %s", query, exc)
+            return ToolResult(
+                result_for_model=(
+                    "WebSearch error: DuckDuckGo blocked this request with a bot check "
+                    "instead of returning results."
+                ),
+                is_error=True,
+            )
+        except Exception as exc:
+            logger.exception("WebSearch failed for query %r", query)
+            return ToolResult(
+                result_for_model=f"WebSearch error: {exc}",
+                is_error=True,
+            )
+
+        if not results:
+            return ToolResult(
+                data={"provider": provider, "query": query, "results": []},
+                result_for_model="No results found.",
+            )
+
+        lines = []
+        for idx, item in enumerate(results, 1):
+            snippet = item["snippet"] or "(no snippet)"
+            lines.append(f"[{idx}] {item['title']}\nURL: {item['url']}\nSnippet: {snippet}")
+
+        return ToolResult(
+            data={"provider": provider, "query": query, "results": results},
+            result_for_model="\n\n".join(lines),
+        )
+
+    async def _has_connectivity(self) -> bool:
+        for probe_url in self._probe_urls():
+            if await self._probe_endpoint(probe_url):
+                return True
+        return False
+
+    def _probe_urls(self) -> list[str]:
+        if self._provider == "tavily":
+            return [_TAVILY_URL]
+        if self._provider == "ddg":
+            return [_DDG_BASE_URL]
+        if self._api_key:
+            return [_TAVILY_URL, _DDG_BASE_URL, _BING_HOME_URL]
+        return [_DDG_BASE_URL, _BING_HOME_URL]
+
+    async def _probe_endpoint(self, url: str) -> bool:
+        try:
+            async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
+                response = await client.get(url)
+            return response.status_code < 500
+        except Exception:
+            return False
+
+    async def _run_search(self, query: str, num_results: int) -> tuple[str, list[dict[str, str]]]:
+        if self._provider == "tavily":
+            return "tavily", await self._search_tavily(query, num_results)
+        if self._provider == "ddg":
+            if _ddg_in_backoff():
+                raise DuckDuckGoUnavailable("DuckDuckGo is still blocking this client")
+            return "ddg", await self._search_ddg(query, num_results)
+
+        if self._api_key:
+            try:
+                return "tavily", await self._search_tavily(query, num_results)
+            except Exception:
+                logger.warning("Tavily search failed; retrying with DuckDuckGo", exc_info=True)
+
+        return await self._search_ddg_then_bing(query, num_results)
+
+    async def _search_ddg_then_bing(self, query: str, num_results: int) -> tuple[str, list[dict[str, str]]]:
+        """Use DuckDuckGo, and switch to Bing when DuckDuckGo serves a bot check."""
+        if not _ddg_in_backoff():
+            try:
+                return "ddg", await self._search_ddg(query, num_results)
+            except DuckDuckGoUnavailable:
+                logger.warning("DuckDuckGo blocked WebSearch; retrying with Bing")
+        return "bing", await self._search_bing(query, num_results)
+
+    async def _search_tavily(self, query: str, num_results: int) -> list[dict[str, str]]:
+        if not self._api_key:
+            raise RuntimeError(f"{self._api_key_env} is not configured")
+
+        payload = {
+            "api_key": self._api_key,
+            "query": query,
+            "max_results": num_results,
+            "search_depth": "basic",
+            "include_answer": False,
+            "include_raw_content": False,
+        }
+        async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
+            response = await client.post(_TAVILY_URL, json=payload)
+            response.raise_for_status()
+            data = response.json()
+
+        raw_results = data.get("results", [])
+        return self._normalize_results(
+            {
+                "title": item.get("title", ""),
+                "url": item.get("url", ""),
+                "snippet": item.get("content", "") or item.get("snippet", ""),
+            }
+            for item in raw_results
+        )[:num_results]
+
+    async def _search_ddg(self, query: str, num_results: int) -> list[dict[str, str]]:
+        url = f"{_DDG_BASE_URL}?q={quote(query)}"
+        headers = {
+            "User-Agent": _BROWSER_UA,
+            "Accept": "text/html,application/xhtml+xml",
+            "Accept-Language": "en-US,en;q=0.9",
+        }
+        try:
+            async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True, headers=headers) as client:
+                response = await client.get(url)
+        except httpx.HTTPError as exc:
+            _note_ddg_unavailable()
+            raise DuckDuckGoUnavailable(f"DuckDuckGo request failed: {exc}") from exc
+
+        if _is_ddg_challenge(response.status_code, response.text):
+            _note_ddg_unavailable()
+            raise DuckDuckGoUnavailable("DuckDuckGo returned a bot check instead of results")
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            _note_ddg_unavailable()
+            raise DuckDuckGoUnavailable(f"DuckDuckGo returned HTTP {response.status_code}") from exc
+        return self._normalize_results(_parse_ddg_html(response.text))[:num_results]
+
+    async def _search_bing(self, query: str, num_results: int) -> list[dict[str, str]]:
+        headers = {
+            "User-Agent": _BROWSER_UA,
+            "Accept": "application/rss+xml, application/xml, text/xml",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        }
+        async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True, headers=headers) as client:
+            response = await client.get(_BING_SEARCH_URL, params={"q": query, "format": "rss"})
+            response.raise_for_status()
+            xml_text = response.text
+        return self._normalize_results(_parse_bing_rss(xml_text))[:num_results]
+
+    def _normalize_results(self, items: Any) -> list[dict[str, str]]:
+        results: list[dict[str, str]] = []
+        seen_urls: set[str] = set()
+        for item in items:
+            title = _clean_text(str(item.get("title", "")))
+            url = str(item.get("url", "")).strip()
+            snippet = _clean_text(str(item.get("snippet", "")))
+            if not title or not _is_http_url(url) or url in seen_urls:
+                continue
+            seen_urls.add(url)
+            results.append({"title": title, "url": url, "snippet": snippet})
+        return results

crabcode_core/turn_details.py

@@ -0,0 +1,64 @@
+"""Per-input statistics, independent of client connections and compaction."""
+
+from datetime import datetime, timezone
+from time import monotonic
+from typing import Any
+
+from crabcode_core.types.event import (
+    CompactEvent, CoreEvent, StreamModeEvent, StreamRetryEvent, StreamTextEvent,
+    ThinkingEvent, ToolUseEvent, TurnCompleteEvent,
+)
+
+
+class TurnDetailsTracker:
+    def __init__(self) -> None:
+        self.started_at = datetime.now(timezone.utc).isoformat()
+        self.started_clock = monotonic()
+        self.tool_ids: set[str] = set()
+        self.thinking_count = 0
+        self.request_count = 0
+        self.retry_count = 0
+        self.compact_count = 0
+        self.thinking = False
+
+    def observe(self, event: CoreEvent) -> None:
+        # Managed-agent activity has its own transcript and statistics.
+        if getattr(event, "agent_id", None) is not None:
+            return
+        if isinstance(event, ThinkingEvent) and event.text:
+            if not self.thinking:
+                self.thinking_count += 1
+            self.thinking = True
+        elif isinstance(event, (StreamTextEvent, ToolUseEvent)):
+            self.thinking = False
+            if isinstance(event, ToolUseEvent):
+                self.tool_ids.add(event.tool_use_id)
+        elif isinstance(event, StreamModeEvent):
+            if event.mode != "thinking":
+                self.thinking = False
+            if event.mode == "requesting":
+                self.request_count += 1
+        elif isinstance(event, StreamRetryEvent):
+            self.retry_count += 1
+            self.thinking = False
+        elif isinstance(event, CompactEvent):
+            self.compact_count += 1
+            self.thinking = False
+
+    def finish(self, event: TurnCompleteEvent, *, session_id: str, model: str, provider: str) -> dict[str, Any]:
+        return {
+            "session_id": session_id,
+            "started_at": self.started_at,
+            "ended_at": datetime.now(timezone.utc).isoformat(),
+            "duration_ms": max(0, round((monotonic() - self.started_clock) * 1000)),
+            "tool_call_count": len(self.tool_ids),
+            "thinking_count": self.thinking_count,
+            "request_count": self.request_count,
+            "retry_count": self.retry_count,
+            "compact_count": self.compact_count,
+            "model": model,
+            "provider": provider,
+            "reason": event.reason,
+            "usage": dict(event.usage),
+            "source": "recorded",
+        }

crabcode_core/types/__init__.py

@@ -0,0 +1,95 @@
+"""Core type definitions for CrabCode."""
+
+from crabcode_core.types.message import (
+    AssistantMessage,
+    ContentBlock,
+    ImageBlock,
+    Message,
+    MessageRole,
+    SystemMessage,
+    TextBlock,
+    ThinkingBlock,
+    ToolResultBlock,
+    ToolUseBlock,
+    UserMessage,
+)
+from crabcode_core.types.tool import (
+    PermissionBehavior,
+    PermissionResult,
+    Tool,
+    ToolContext,
+    ToolResult,
+)
+from crabcode_core.types.event import (
+    AgentOutputEvent,
+    AgentStateEvent,
+    CompactEvent,
+    CoreEvent,
+    DocumentJobEvent,
+    ErrorEvent,
+    PermissionRequestEvent,
+    PermissionResponseEvent,
+    PeerMessageEvent,
+    StreamRetryEvent,
+    StreamTextEvent,
+    ThinkingEvent,
+    ToolResultEvent,
+    ToolUseEvent,
+    TurnCompleteEvent,
+)
+from crabcode_core.types.config import (
+    AgentSettings,
+    AgentTypeConfig,
+    AiReviewSettings,
+    ApiConfig,
+    CrabCodeSettings,
+    CrossSessionSettings,
+    LoggingSettings,
+    McpServerConfig,
+    PermissionRule,
+    PermissionsSettings,
+)
+
+__all__ = [
+    "AssistantMessage",
+    "ContentBlock",
+    "ImageBlock",
+    "Message",
+    "MessageRole",
+    "SystemMessage",
+    "TextBlock",
+    "ThinkingBlock",
+    "ToolResultBlock",
+    "ToolUseBlock",
+    "UserMessage",
+    "PermissionBehavior",
+    "PermissionResult",
+    "Tool",
+    "ToolContext",
+    "ToolResult",
+    "CompactEvent",
+    "AgentOutputEvent",
+    "AgentStateEvent",
+    "CoreEvent",
+    "DocumentJobEvent",
+    "ErrorEvent",
+    "PermissionRequestEvent",
+    "PermissionResponseEvent",
+    "PeerMessageEvent",
+    "StreamRetryEvent",
+    "StreamTextEvent",
+    "ThinkingEvent",
+    "ToolResultEvent",
+    "ToolUseEvent",
+    "TurnCompleteEvent",
+    "ApiConfig",
+    "AgentSettings",
+    "AgentTypeConfig",
+    "AiReviewSettings",
+    "CrabCodeSettings",
+    "CrossSessionSettings",
+    "LoggingSettings",
+    "McpServerConfig",
+    "PermissionRule",
+    "PermissionsSettings",
+]