klaude-code 1.2.6__py3-none-any.whl → 1.8.0__py3-none-any.whl

klaude_code/core/tool/web/web_fetch_tool.md

@@ -5,4 +5,4 @@ The tool automatically processes the response based on Content-Type:
 - JSON responses are formatted with indentation
 - Markdown and other text content is returned as-is
 
-Use this tool to retrieve web page content for analysis.
+Content is always saved to a local file. The file path is included at the start of the output in a `<file_saved>` tag. For large content that gets truncated, you can read the saved file directly.

klaude_code/core/tool/web/web_fetch_tool.py

@@ -1,29 +1,89 @@
 import asyncio
 import json
+import re
+import time
 import urllib.error
 import urllib.request
 from http.client import HTTPResponse
 from pathlib import Path
+from urllib.parse import quote, urlparse, urlunparse
 
 from pydantic import BaseModel
 
-from klaude_code.core.tool.tool_abc import ToolABC, load_desc
+from klaude_code import const
+from klaude_code.core.tool.tool_abc import ToolABC, ToolConcurrencyPolicy, ToolMetadata, load_desc
 from klaude_code.core.tool.tool_registry import register
 from klaude_code.protocol import llm_param, model, tools
 
 DEFAULT_TIMEOUT_SEC = 30
 DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; KlaudeCode/1.0)"
+WEB_FETCH_SAVE_DIR = Path(const.TOOL_OUTPUT_TRUNCATION_DIR) / "web"
 
 
-def _extract_content_type(response: HTTPResponse) -> str:
-    """Extract the base content type without charset parameters."""
-    content_type = response.getheader("Content-Type", "")
-    return content_type.split(";")[0].strip().lower()
+def _encode_url(url: str) -> str:
+    """Encode non-ASCII characters in URL to make it safe for HTTP requests."""
+    parsed = urlparse(url)
+    encoded_path = quote(parsed.path, safe="/-_.~")
+    encoded_query = quote(parsed.query, safe="=&-_.~")
+    # Handle IDN (Internationalized Domain Names) by encoding to punycode
+    try:
+        netloc = parsed.netloc.encode("idna").decode("ascii")
+    except UnicodeError:
+        netloc = parsed.netloc
+    return urlunparse((parsed.scheme, netloc, encoded_path, parsed.params, encoded_query, parsed.fragment))
+
+
+def _extract_content_type_and_charset(response: HTTPResponse) -> tuple[str, str | None]:
+    """Extract the base content type and charset from Content-Type header."""
+    content_type_header = response.getheader("Content-Type", "")
+    parts = content_type_header.split(";")
+    content_type = parts[0].strip().lower()
+
+    charset = None
+    for part in parts[1:]:
+        part = part.strip()
+        if part.lower().startswith("charset="):
+            charset = part[8:].strip().strip("\"'")
+            break
+
+    return content_type, charset
+
+
+def _detect_encoding(data: bytes, declared_charset: str | None) -> str:
+    """Detect the encoding of the data."""
+    # 1. Use declared charset from HTTP header if available
+    if declared_charset:
+        return declared_charset
 
+    # 2. Try to detect from HTML meta tags (check first 2KB)
+    head = data[:2048].lower()
+    # <meta charset="xxx">
+    if match := re.search(rb'<meta[^>]+charset=["\']?([^"\'\s>]+)', head):
+        return match.group(1).decode("ascii", errors="ignore")
+    # <meta http-equiv="Content-Type" content="text/html; charset=xxx">
+    if match := re.search(rb'content=["\'][^"\']*charset=([^"\'\s;]+)', head):
+        return match.group(1).decode("ascii", errors="ignore")
 
-def _validate_utf8(data: bytes) -> str:
-    """Validate and decode bytes as UTF-8."""
-    return data.decode("utf-8")
+    # 3. Use chardet for automatic detection
+    import chardet
+
+    result = chardet.detect(data)
+    if result["encoding"] and result["confidence"] and result["confidence"] > 0.7:
+        return result["encoding"]
+
+    # 4. Default to UTF-8
+    return "utf-8"
+
+
+def _decode_content(data: bytes, declared_charset: str | None) -> str:
+    """Decode bytes to string with automatic encoding detection."""
+    encoding = _detect_encoding(data, declared_charset)
+
+    try:
+        return data.decode(encoding)
+    except (UnicodeDecodeError, LookupError):
+        # Fallback: try UTF-8 with replacement for invalid chars
+        return data.decode("utf-8", errors="replace")
 
 
 def _convert_html_to_markdown(html: str) -> str:
@@ -43,6 +103,30 @@ def _format_json(text: str) -> str:
         return text
 
 
+def _extract_url_filename(url: str) -> str:
+    """Extract a safe filename from a URL."""
+    parsed = urlparse(url)
+    host = parsed.netloc.replace(".", "_").replace(":", "_")
+    path = parsed.path.strip("/").replace("/", "_")
+    name = f"{host}_{path}" if path else host
+    name = re.sub(r"[^a-zA-Z0-9_\-]", "_", name)
+    return name[:80] if len(name) > 80 else name
+
+
+def _save_web_content(url: str, content: str) -> str | None:
+    """Save web content to file. Returns file path or None on failure."""
+    try:
+        WEB_FETCH_SAVE_DIR.mkdir(parents=True, exist_ok=True)
+        timestamp = int(time.time())
+        identifier = _extract_url_filename(url)
+        filename = f"{identifier}-{timestamp}.md"
+        file_path = WEB_FETCH_SAVE_DIR / filename
+        file_path.write_text(content, encoding="utf-8")
+        return str(file_path)
+    except OSError:
+        return None
+
+
 def _process_content(content_type: str, text: str) -> str:
     """Process content based on Content-Type header."""
     if content_type == "text/html":
@@ -69,17 +153,22 @@ def _fetch_url(url: str, timeout: int = DEFAULT_TIMEOUT_SEC) -> tuple[str, str]:
         "Accept": "text/markdown, */*",
         "User-Agent": DEFAULT_USER_AGENT,
     }
-    request = urllib.request.Request(url, headers=headers)
+    encoded_url = _encode_url(url)
+    request = urllib.request.Request(encoded_url, headers=headers)
 
     with urllib.request.urlopen(request, timeout=timeout) as response:
-        content_type = _extract_content_type(response)
+        content_type, charset = _extract_content_type_and_charset(response)
         data = response.read()
-        text = _validate_utf8(data)
+        text = _decode_content(data, charset)
         return content_type, text
 
 
 @register(tools.WEB_FETCH)
 class WebFetchTool(ToolABC):
+    @classmethod
+    def metadata(cls) -> ToolMetadata:
+        return ToolMetadata(concurrency_policy=ToolConcurrencyPolicy.CONCURRENT, has_side_effects=True)
+
     @classmethod
     def schema(cls) -> llm_param.ToolSchema:
         return llm_param.ToolSchema(
@@ -120,40 +209,41 @@ class WebFetchTool(ToolABC):
         if not url.startswith(("http://", "https://")):
             return model.ToolResultItem(
                 status="error",
-                output="Invalid URL: must start with http:// or https://",
+                output=f"Invalid URL: must start with http:// or https:// (url={url})",
             )
 
         try:
             content_type, text = await asyncio.to_thread(_fetch_url, url)
             processed = _process_content(content_type, text)
 
+            # Always save content to file
+            saved_path = _save_web_content(url, processed)
+
+            # Build output with file path info
+            output = f"<file_saved>{saved_path}</file_saved>\n\n{processed}" if saved_path else processed
+
             return model.ToolResultItem(
                 status="success",
-                output=processed,
+                output=output,
             )
 
         except urllib.error.HTTPError as e:
             return model.ToolResultItem(
                 status="error",
-                output=f"HTTP error {e.code}: {e.reason}",
+                output=f"HTTP error {e.code}: {e.reason} (url={url})",
             )
         except urllib.error.URLError as e:
             return model.ToolResultItem(
                 status="error",
-                output=f"URL error: {e.reason}",
-            )
-        except UnicodeDecodeError as e:
-            return model.ToolResultItem(
-                status="error",
-                output=f"Content is not valid UTF-8: {e}",
+                output=f"URL error: {e.reason} (url={url})",
             )
         except TimeoutError:
             return model.ToolResultItem(
                 status="error",
-                output=f"Request timed out after {DEFAULT_TIMEOUT_SEC} seconds",
+                output=f"Request timed out after {DEFAULT_TIMEOUT_SEC} seconds (url={url})",
             )
         except Exception as e:
             return model.ToolResultItem(
                 status="error",
-                output=f"Failed to fetch URL: {e}",
+                output=f"Failed to fetch URL: {e} (url={url})",
             )

klaude_code/core/tool/web/web_search_tool.md

@@ -0,0 +1,23 @@
+- Search the web and use the results to inform responses
+- Provides up-to-date information for current events and recent data
+- Returns search result information formatted as search result blocks, including links as markdown hyperlinks
+- Use this tool for accessing information beyond your knowledge cutoff
+- Searches are performed automatically within a single API call
+
+CRITICAL REQUIREMENT - You MUST follow this:
+- After answering the user's question, you MUST include a "Sources:" section at the end of your response
+- In the Sources section, list all relevant URLs from the search results as markdown hyperlinks: [Title](URL)
+- This is MANDATORY - never skip including sources in your response
+- Example format:
+
+[Your answer here]
+
+Sources:
+- [Source Title 1](https://example.com/1)
+- [Source Title 2](https://example.com/2)
+
+Usage notes:
+- Domain filtering is supported to include or block specific websites
+- Web search is only available in the US
+- Account for "Today's date" in <env>. For example, if <env> says "Today's date: 2025-07-01", and the user wants the latest docs, do not use 2024 in the search query. Use 2025.
+

klaude_code/core/tool/web/web_search_tool.py

@@ -0,0 +1,130 @@
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from klaude_code.core.tool.tool_abc import ToolABC, ToolConcurrencyPolicy, ToolMetadata, load_desc
+from klaude_code.core.tool.tool_registry import register
+from klaude_code.protocol import llm_param, model, tools
+
+DEFAULT_MAX_RESULTS = 10
+MAX_RESULTS_LIMIT = 20
+
+
+@dataclass
+class SearchResult:
+    """A single search result from DuckDuckGo."""
+
+    title: str
+    url: str
+    snippet: str
+    position: int
+
+
+def _search_duckduckgo(query: str, max_results: int) -> list[SearchResult]:
+    """Perform a web search using ddgs library."""
+    from ddgs import DDGS  # type: ignore
+
+    results: list[SearchResult] = []
+
+    with DDGS() as ddgs:
+        for i, r in enumerate(ddgs.text(query, max_results=max_results)):
+            results.append(
+                SearchResult(
+                    title=r.get("title", ""),
+                    url=r.get("href", ""),
+                    snippet=r.get("body", ""),
+                    position=i + 1,
+                )
+            )
+
+    return results
+
+
+def _format_results(results: list[SearchResult]) -> str:
+    """Format search results for LLM consumption."""
+    if not results:
+        return (
+            "No results were found for your search query. "
+            "Please try rephrasing your search or using different keywords."
+        )
+
+    lines = [f"Found {len(results)} search results:\n"]
+
+    for result in results:
+        lines.append(f"{result.position}. {result.title}")
+        lines.append(f"   URL: {result.url}")
+        lines.append(f"   Summary: {result.snippet}\n")
+
+    return "\n".join(lines)
+
+
+@register(tools.WEB_SEARCH)
+class WebSearchTool(ToolABC):
+    @classmethod
+    def metadata(cls) -> ToolMetadata:
+        return ToolMetadata(concurrency_policy=ToolConcurrencyPolicy.CONCURRENT, has_side_effects=False)
+
+    @classmethod
+    def schema(cls) -> llm_param.ToolSchema:
+        return llm_param.ToolSchema(
+            name=tools.WEB_SEARCH,
+            type="function",
+            description=load_desc(Path(__file__).parent / "web_search_tool.md"),
+            parameters={
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "The search query to use",
+                    },
+                    "max_results": {
+                        "type": "integer",
+                        "description": f"Maximum number of results to return (default: {DEFAULT_MAX_RESULTS}, max: {MAX_RESULTS_LIMIT})",
+                    },
+                },
+                "required": ["query"],
+            },
+        )
+
+    class WebSearchArguments(BaseModel):
+        query: str
+        max_results: int = DEFAULT_MAX_RESULTS
+
+    @classmethod
+    async def call(cls, arguments: str) -> model.ToolResultItem:
+        try:
+            args = WebSearchTool.WebSearchArguments.model_validate_json(arguments)
+        except ValueError as e:
+            return model.ToolResultItem(
+                status="error",
+                output=f"Invalid arguments: {e}",
+            )
+        return await cls.call_with_args(args)
+
+    @classmethod
+    async def call_with_args(cls, args: WebSearchArguments) -> model.ToolResultItem:
+        query = args.query.strip()
+        if not query:
+            return model.ToolResultItem(
+                status="error",
+                output="Query cannot be empty",
+            )
+
+        max_results = min(max(args.max_results, 1), MAX_RESULTS_LIMIT)
+
+        try:
+            results = await asyncio.to_thread(_search_duckduckgo, query, max_results)
+            formatted = _format_results(results)
+
+            return model.ToolResultItem(
+                status="success",
+                output=formatted,
+            )
+
+        except Exception as e:
+            return model.ToolResultItem(
+                status="error",
+                output=f"Search failed: {e}",
+            )