voidx 1.0.0__py3-none-any.whl

voidx/tools/todo.py

@@ -0,0 +1,82 @@
+"""TodoWrite tool — stateful task list, Claude Code aligned.
+
+Each call REPLACES the entire list. Status persists across calls via tracker.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from voidx.tools.base import BaseTool, model_to_json_schema, ToolContext, ToolResult
+
+
+class TodoItem(BaseModel):
+    content: str = Field(description="Task description, one sentence.")
+    status: str = Field(
+        default="pending",
+        description="pending | in_progress | completed | cancelled"
+    )
+
+
+class TodoInput(BaseModel):
+    todos: list[TodoItem] = Field(
+        description="Full todo list — replaces the previous list entirely. Include ALL items."
+    )
+
+
+class TodoWriteTool(BaseTool):
+    id = "todo"
+    description = (
+        "Create and manage a task list. Each call REPLACES the entire list — "
+        "pass the full updated list. Items:[{id, status, content}] "
+        "Status: pending → in_progress → completed."
+    )
+
+    def __init__(self, tracker=None):
+        super().__init__()
+        self._tracker = tracker
+
+    def parameters_schema(self) -> dict:
+        return model_to_json_schema(TodoInput)
+
+    async def execute(self, args: dict, ctx: ToolContext) -> ToolResult:
+        inp = TodoInput.model_validate(args)
+
+        total = len(inp.todos)
+        done = sum(1 for t in inp.todos if t.status == "completed")
+        in_progress = sum(1 for t in inp.todos if t.status == "in_progress")
+        pending = sum(1 for t in inp.todos if t.status == "pending")
+        cancelled = sum(1 for t in inp.todos if t.status == "cancelled")
+
+        # Store in tracker if available
+        if self._tracker:
+            self._tracker._todos = inp.todos
+
+        # Format with icons and progress
+        ICONS = {"pending": "○", "in_progress": "◐", "completed": "●", "cancelled": "✕"}
+        lines = []
+
+        # Progress bar
+        if total > 0:
+            pct = done / total
+            bar_len = 20
+            filled = int(bar_len * pct)
+            bar = "█" * filled + "░" * (bar_len - filled)
+            lines.append(f"[{bar}] {done}/{total} done")
+
+        # Group by status
+        for status in ["in_progress", "pending", "completed", "cancelled"]:
+            items = [t for t in inp.todos if t.status == status]
+            if not items:
+                continue
+            for item in items:
+                lines.append(f"  {ICONS[item.status]} {item.content}")
+
+        return ToolResult(
+            title=f"Todo: {done}/{total} done · {in_progress} active · {pending} pending",
+            output="\n".join(lines),
+            metadata={
+                "total": total, "done": done, "in_progress": in_progress,
+                "pending": pending, "cancelled": cancelled,
+            },
+        )

voidx/tools/web_content.py

@@ -0,0 +1,357 @@
+"""Shared helpers for web search/fetch tools."""
+
+from __future__ import annotations
+
+import copy
+import html
+import re
+import time
+from dataclasses import dataclass
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qsl, parse_qs, urlencode, unquote, urlparse, urlunparse
+
+from voidx.tools.base import ToolResult
+
+_TRACKING_PARAMS = {
+    "fbclid",
+    "gclid",
+    "igshid",
+    "mc_cid",
+    "mc_eid",
+    "msclkid",
+    "ref",
+    "spm",
+}
+_SKIP_TAGS = {"script", "style", "noscript", "svg", "nav", "footer", "header", "form"}
+_BLOCK_TAGS = {
+    "article",
+    "aside",
+    "blockquote",
+    "dd",
+    "div",
+    "dl",
+    "dt",
+    "figcaption",
+    "figure",
+    "li",
+    "main",
+    "ol",
+    "p",
+    "pre",
+    "section",
+    "table",
+    "tbody",
+    "td",
+    "th",
+    "thead",
+    "tr",
+    "ul",
+}
+_STOP_WORDS = {
+    "about",
+    "after",
+    "before",
+    "content",
+    "extract",
+    "from",
+    "only",
+    "page",
+    "show",
+    "this",
+    "with",
+}
+
+
+@dataclass
+class _CacheEntry:
+    value: Any
+    expires_at: float
+
+
+class WebToolCache:
+    def __init__(self, max_entries: int = 256) -> None:
+        self._items: dict[str, _CacheEntry] = {}
+        self._max_entries = max_entries
+
+    def get(self, key: str) -> Any | None:
+        entry = self._items.get(key)
+        if entry is None:
+            return None
+        if entry.expires_at < time.monotonic():
+            self._items.pop(key, None)
+            return None
+        return copy.deepcopy(entry.value)
+
+    def set(self, key: str, value: Any, ttl_seconds: float) -> None:
+        if len(self._items) >= self._max_entries:
+            oldest = min(self._items, key=lambda item: self._items[item].expires_at)
+            self._items.pop(oldest, None)
+        self._items[key] = _CacheEntry(copy.deepcopy(value), time.monotonic() + ttl_seconds)
+
+    def clear(self) -> None:
+        self._items.clear()
+
+
+WEB_TOOL_CACHE = WebToolCache()
+
+
+def cached_tool_result(result: ToolResult) -> ToolResult:
+    metadata = dict(result.metadata or {})
+    metadata["cached"] = True
+    return result.model_copy(deep=True, update={"metadata": metadata})
+
+
+def canonicalize_url(url: str) -> str:
+    url = _decode_duckduckgo_url(url.strip())
+    parsed = urlparse(url)
+    scheme = (parsed.scheme or "https").lower()
+    host = (parsed.hostname or "").lower()
+    netloc = host
+    if parsed.port and not (
+        scheme == "http" and parsed.port == 80
+        or scheme == "https" and parsed.port == 443
+    ):
+        netloc = f"{host}:{parsed.port}"
+    query_items = [
+        (key, value)
+        for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+        if not key.lower().startswith("utm_") and key.lower() not in _TRACKING_PARAMS
+    ]
+    query = urlencode(sorted(query_items), doseq=True)
+    path = parsed.path or "/"
+    if path != "/":
+        path = path.rstrip("/")
+    return urlunparse((scheme, netloc, path, "", query, ""))
+
+
+def domain_for_url(url: str) -> str:
+    host = urlparse(canonicalize_url(url)).hostname or ""
+    return host.removeprefix("www.")
+
+
+def matches_domain(url: str, domain: str) -> bool:
+    host = domain_for_url(url)
+    target = domain.lower().strip().removeprefix("www.")
+    return host == target or host.endswith(f".{target}")
+
+
+def normalize_search_results(results: list[dict[str, str]]) -> list[dict[str, Any]]:
+    seen: set[str] = set()
+    normalized: list[dict[str, str]] = []
+    for rank, result in enumerate(results, start=1):
+        raw_url = result.get("url", "").strip()
+        if not raw_url:
+            continue
+        url = canonicalize_url(raw_url)
+        if url in seen:
+            continue
+        seen.add(url)
+        title = _clean_inline_text(result.get("title", "")) or domain_for_url(url) or url
+        snippet = _clean_inline_text(result.get("snippet", ""))
+        normalized.append({
+            "rank": rank,
+            "title": title,
+            "url": url,
+            "domain": domain_for_url(url),
+            "snippet": snippet,
+        })
+    return normalized
+
+
+def extract_readable_content(
+    *,
+    url: str,
+    text: str,
+    content_type: str = "",
+    prompt: str = "",
+    max_chars: int = 12_000,
+) -> dict[str, Any]:
+    if "html" in content_type.lower() or _looks_like_html(text):
+        parsed = _ReadableHtmlParser()
+        parsed.feed(text)
+        parsed.close()
+        title = _clean_inline_text(parsed.title)
+        blocks = _dedupe_blocks(parsed.blocks)
+    else:
+        title = ""
+        blocks = _plain_text_blocks(text)
+
+    page_text = "\n\n".join(blocks).strip()
+    relevant = _relevant_blocks(blocks, prompt)
+    if relevant:
+        page_text = "## Relevant excerpts\n\n" + "\n\n".join(relevant) + "\n\n## Page content\n\n" + page_text
+
+    total_chars = len(page_text)
+    truncated = total_chars > max_chars
+    if truncated:
+        page_text = page_text[:max_chars].rstrip()
+        page_text += f"\n\n[truncated: {total_chars} extracted chars, showing first {max_chars}]"
+
+    return {
+        "title": title,
+        "url": canonicalize_url(url),
+        "content": page_text,
+        "total_chars": total_chars,
+        "truncated": truncated,
+        "excerpt_count": len(relevant),
+    }
+
+
+def search_cache_key(
+    *,
+    query: str,
+    allowed_domains: list[str] | None,
+    blocked_domains: list[str] | None,
+    max_results: int,
+    backend: str,
+) -> str:
+    return "|".join([
+        "search:v2",
+        backend,
+        _clean_inline_text(query).lower(),
+        ",".join(sorted((allowed_domains or []))),
+        ",".join(sorted((blocked_domains or []))),
+        str(max_results),
+    ])
+
+
+def fetch_cache_key(url: str, prompt: str, max_chars: int) -> str:
+    return "|".join(["fetch:v2", canonicalize_url(url), _clean_inline_text(prompt).lower(), str(max_chars)])
+
+
+class _ReadableHtmlParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.blocks: list[str] = []
+        self.title = ""
+        self._title_parts: list[str] = []
+        self._current_parts: list[str] = []
+        self._current_tag = ""
+        self._heading_level = 0
+        self._skip_depth = 0
+        self._in_title = False
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS:
+            self._skip_depth += 1
+            return
+        if tag == "title":
+            self._in_title = True
+            self._title_parts = []
+            return
+        if self._skip_depth:
+            return
+        if tag == "br":
+            self._append_text("\n")
+            return
+        if tag in _BLOCK_TAGS or re.fullmatch(r"h[1-6]", tag):
+            self._finish_block()
+            self._current_tag = tag
+            self._heading_level = int(tag[1]) if re.fullmatch(r"h[1-6]", tag) else 0
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS and self._skip_depth:
+            self._skip_depth -= 1
+            return
+        if tag == "title":
+            self.title = _clean_inline_text(" ".join(self._title_parts))
+            self._in_title = False
+            return
+        if self._skip_depth:
+            return
+        if tag == self._current_tag or tag in _BLOCK_TAGS or re.fullmatch(r"h[1-6]", tag):
+            self._finish_block()
+
+    def handle_data(self, data: str) -> None:
+        if self._skip_depth:
+            return
+        if self._in_title:
+            self._title_parts.append(data)
+            return
+        if data.strip():
+            self._append_text(data)
+
+    def close(self) -> None:
+        self._finish_block()
+        super().close()
+
+    def _append_text(self, text: str) -> None:
+        if not self._current_tag:
+            self._current_tag = "p"
+        self._current_parts.append(text)
+
+    def _finish_block(self) -> None:
+        raw = " ".join(self._current_parts)
+        text = _clean_block_text(raw)
+        if text:
+            if self._heading_level:
+                text = f"{'#' * self._heading_level} {text}"
+            elif self._current_tag == "li":
+                text = f"- {text}"
+            self.blocks.append(text)
+        self._current_parts = []
+        self._current_tag = ""
+        self._heading_level = 0
+
+
+def _decode_duckduckgo_url(url: str) -> str:
+    parsed = urlparse(url)
+    if "duckduckgo.com" not in (parsed.hostname or ""):
+        return url
+    values = parse_qs(parsed.query).get("uddg")
+    if values:
+        return unquote(values[0])
+    return url
+
+
+def _looks_like_html(text: str) -> bool:
+    head = text[:500].lower()
+    return "<html" in head or "<body" in head or "<!doctype html" in head
+
+
+def _plain_text_blocks(text: str) -> list[str]:
+    return [
+        _clean_block_text(block)
+        for block in re.split(r"\n\s*\n", text)
+        if _clean_block_text(block)
+    ]
+
+
+def _dedupe_blocks(blocks: list[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for block in blocks:
+        normalized = _clean_inline_text(block).lower()
+        if len(normalized) < 2 or normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(block)
+    return result
+
+
+def _relevant_blocks(blocks: list[str], prompt: str) -> list[str]:
+    terms = [
+        term.lower()
+        for term in re.findall(r"[A-Za-z0-9_.-]{3,}|[\u4e00-\u9fff]{2,}", prompt)
+        if term.lower() not in _STOP_WORDS
+    ]
+    if not terms:
+        return []
+    matches = [
+        block
+        for block in blocks
+        if any(term in block.lower() for term in terms)
+    ]
+    return matches[:5]
+
+
+def _clean_inline_text(text: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(str(text))).strip()
+
+
+def _clean_block_text(text: str) -> str:
+    lines = [_clean_inline_text(line) for line in str(text).splitlines()]
+    return "\n".join(line for line in lines if line).strip()

voidx/tools/web_mcp.py

@@ -0,0 +1,107 @@
+"""MCP delegation helpers for web tools."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from voidx.tools.base import ToolContext, ToolResult
+
+
+def adapt_mcp_web_arguments(kind: str, tool: str, arguments: dict[str, Any]) -> dict[str, Any]:
+    if kind == "search" and tool == "tavily_search":
+        return _adapt_tavily_search_arguments(arguments)
+    if kind == "fetch" and tool == "tavily_extract":
+        return _adapt_tavily_extract_arguments(arguments)
+    return dict(arguments)
+
+
+def _adapt_tavily_search_arguments(arguments: dict[str, Any]) -> dict[str, Any]:
+    mapped: dict[str, Any] = {"query": arguments["query"]}
+
+    allowed = arguments.get("allowed_domains")
+    if allowed:
+        mapped["include_domains"] = allowed
+
+    blocked = arguments.get("blocked_domains")
+    if blocked:
+        mapped["exclude_domains"] = blocked
+
+    max_results = arguments.get("max_results")
+    if max_results:
+        mapped["max_results"] = max_results
+
+    return mapped
+
+
+def _adapt_tavily_extract_arguments(arguments: dict[str, Any]) -> dict[str, Any]:
+    mapped: dict[str, Any] = {"urls": [arguments["url"]]}
+
+    prompt = arguments.get("prompt")
+    if prompt:
+        mapped["query"] = prompt
+
+    return mapped
+
+
+async def call_mcp_web_tool(
+    *,
+    kind: str,
+    settings: Any,
+    ctx: ToolContext,
+    arguments: dict[str, Any],
+    title: str,
+) -> ToolResult | None:
+    if settings is None or not hasattr(settings, "get_web_tool_route"):
+        return None
+
+    route = settings.get_web_tool_route(kind)
+    if route.backend != "mcp":
+        return None
+
+    if not route.server or not route.tool:
+        return ToolResult(
+            output=f"Web {kind} is configured for MCP but no server/tool route is set.",
+            metadata={"backend": "mcp", "error": True, "kind": kind},
+        )
+
+    manager = getattr(ctx, "mcp_manager", None)
+    if manager is None:
+        return ToolResult(
+            output=f"Web {kind} is configured for MCP but no MCP manager is available.",
+            metadata={
+                "backend": "mcp",
+                "error": True,
+                "kind": kind,
+                "server": route.server,
+                "tool": route.tool,
+            },
+        )
+
+    try:
+        adapted_arguments = adapt_mcp_web_arguments(kind, route.tool, arguments)
+        result = await manager.call_tool(route.server, route.tool, adapted_arguments)
+    except Exception as exc:
+        return ToolResult(
+            output=f"MCP web {kind} failed via {route.server}/{route.tool}: {exc}",
+            metadata={
+                "backend": "mcp",
+                "error": True,
+                "kind": kind,
+                "server": route.server,
+                "tool": route.tool,
+            },
+        )
+
+    from voidx.mcp.tool import format_mcp_call_result
+
+    return ToolResult(
+        title=title,
+        output=format_mcp_call_result(result),
+        metadata={
+            "backend": "mcp",
+            "kind": kind,
+            "server": route.server,
+            "tool": route.tool,
+            "error": result.isError,
+        },
+    )

voidx/tools/webfetch.py

@@ -0,0 +1,155 @@
+"""WebFetch tool — fetch web page content. SSRF-protected."""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+import httpx
+from pydantic import BaseModel, Field
+
+from voidx.tools.base import BaseTool, model_to_json_schema, ToolContext, ToolResult
+from voidx.tools.web_content import (
+    WEB_TOOL_CACHE,
+    cached_tool_result,
+    canonicalize_url,
+    extract_readable_content,
+    fetch_cache_key,
+)
+from voidx.tools.web_mcp import call_mcp_web_tool
+
+_PRIVATE_RANGES = (
+    ipaddress.IPv4Network("127.0.0.0/8"),      # loopback
+    ipaddress.IPv4Network("10.0.0.0/8"),       # private
+    ipaddress.IPv4Network("172.16.0.0/12"),    # private
+    ipaddress.IPv4Network("192.168.0.0/16"),   # private
+    ipaddress.IPv4Network("169.254.0.0/16"),   # link-local
+    ipaddress.IPv4Network("0.0.0.0/8"),        # current network
+    ipaddress.IPv6Network("::1/128"),          # IPv6 loopback
+    ipaddress.IPv6Network("fc00::/7"),         # IPv6 unique local
+    ipaddress.IPv6Network("fe80::/10"),        # IPv6 link-local
+)
+
+
+def _is_private_host(host: str) -> bool:
+    """Check if a hostname or IP resolves to a private/internal address."""
+    try:
+        addr = ipaddress.ip_address(host)
+    except ValueError:
+        try:
+            addr = ipaddress.ip_address(socket.gethostbyname(host))
+        except (socket.gaierror, OSError):
+            return False
+    return any(addr in net for net in _PRIVATE_RANGES)
+
+
+class WebFetchInput(BaseModel):
+    url: str = Field(description="URL to fetch content from")
+    prompt: str = Field(description="The prompt to run on the fetched content")
+    max_chars: int = Field(default=12000, ge=1000, le=50000, description="Maximum extracted text characters")
+
+
+@dataclass
+class _FetchResponse:
+    url: str
+    status_code: int
+    text: str
+    content_type: str
+
+
+class WebFetchTool(BaseTool):
+    id = "webfetch"
+    description = "Fetch content from a URL and convert to readable text."
+
+    def __init__(self, settings=None):
+        self._settings = settings
+
+    def parameters_schema(self) -> dict:
+        return model_to_json_schema(WebFetchInput)
+
+    async def execute(self, args: dict, ctx: ToolContext) -> ToolResult:
+        inp = WebFetchInput.model_validate(args)
+        mcp_arguments = inp.model_dump(exclude_none=True)
+        if "max_chars" not in args:
+            mcp_arguments.pop("max_chars", None)
+        mcp_result = await call_mcp_web_tool(
+            kind="fetch",
+            settings=self._settings,
+            ctx=ctx,
+            arguments=mcp_arguments,
+            title=f"Fetched: {inp.url}",
+        )
+        if mcp_result is not None:
+            return mcp_result
+
+        parsed = urlparse(inp.url)
+        if parsed.scheme not in {"http", "https"}:
+            return ToolResult(
+                output=f"Blocked: unsupported URL scheme '{parsed.scheme or '(none)'}'",
+                metadata={"url": inp.url, "blocked": True, "reason": "unsupported_scheme"},
+            )
+        if parsed.hostname and _is_private_host(parsed.hostname):
+            return ToolResult(
+                output=f"Blocked: {parsed.hostname} resolves to a private/internal address",
+                metadata={"url": inp.url, "blocked": True},
+            )
+
+        key = fetch_cache_key(inp.url, inp.prompt, inp.max_chars)
+        cached = WEB_TOOL_CACHE.get(key)
+        if isinstance(cached, ToolResult):
+            return cached_tool_result(cached)
+
+        try:
+            resp = await _fetch_url(inp.url)
+            final_host = urlparse(resp.url).hostname
+            if final_host and _is_private_host(final_host):
+                return ToolResult(
+                    output=f"Blocked: redirect target {final_host} resolves to a private/internal address",
+                    metadata={"url": inp.url, "final_url": resp.url, "blocked": True},
+                )
+            extracted = extract_readable_content(
+                url=resp.url,
+                text=resp.text,
+                content_type=resp.content_type,
+                prompt=inp.prompt,
+                max_chars=inp.max_chars,
+            )
+            output = extracted["content"] or resp.text[:inp.max_chars]
+
+            result = ToolResult(
+                title=f"Fetched: {canonicalize_url(resp.url)}",
+                output=output,
+                metadata={
+                    "url": inp.url,
+                    "canonical_url": extracted["url"],
+                    "status": resp.status_code,
+                    "size": len(resp.text),
+                    "content_type": resp.content_type,
+                    "title": extracted["title"],
+                    "extracted_chars": extracted["total_chars"],
+                    "truncated": extracted["truncated"],
+                    "excerpt_count": extracted["excerpt_count"],
+                    "cached": False,
+                },
+            )
+            WEB_TOOL_CACHE.set(key, result, ttl_seconds=1800)
+            return result
+        except Exception as e:
+            return ToolResult(
+                output=f"Failed to fetch {inp.url}: {e}",
+                metadata={"url": inp.url, "error": str(e)},
+            )
+
+
+async def _fetch_url(url: str) -> _FetchResponse:
+    async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client:
+        resp = await client.get(url, headers={"User-Agent": "voidx/0.1"})
+        resp.raise_for_status()
+        return _FetchResponse(
+            url=str(resp.url),
+            status_code=resp.status_code,
+            text=resp.text,
+            content_type=resp.headers.get("content-type", ""),
+        )