illusion-code 0.1.0__py3-none-any.whl

illusion/tools/tool_search_tool.py

@@ -0,0 +1,156 @@
+"""
+工具搜索工具
+============
+
+本模块提供搜索可用工具注册表的功能，支持精确名称查询和关键词搜索，
+返回匹配工具的完整 JSONSchema 定义。
+
+主要组件：
+    - ToolSearchTool: 搜索工具注册表的工具
+
+使用示例：
+    >>> from illusion.tools import ToolSearchTool
+    >>> tool = ToolSearchTool()
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from illusion.tools.base import BaseTool, ToolExecutionContext, ToolResult
+
+
+class ToolSearchToolInput(BaseModel):
+    """工具搜索参数。
+
+    属性：
+        query: 在工具名称和描述中搜索的子字符串，支持特殊查询语法：
+            - "select:Tool1,Tool2" — 按名称精确获取
+            - "+term other" — 要求名称包含 term，按剩余词排序
+            - "keyword list" — 关键词搜索，按匹配度排序
+    """
+
+    query: str = Field(description="Substring to search in tool names and descriptions")
+
+
+class ToolSearchTool(BaseTool):
+    """搜索工具注册表内容并返回匹配工具的完整 schema 定义。
+
+    支持三种查询模式：
+    1. select: 前缀 — 按逗号分隔的名称精确匹配
+    2. + 前缀 — 要求第一个词出现在工具名称中，按剩余词排名
+    3. 普通关键词 — 按匹配度排序返回最佳结果
+    """
+
+    name = "tool_search"
+    description = """Fetches full schema definitions for deferred tools so they can be called.
+
+Deferred tools appear by name in <system-reminder> messages. Until fetched, only the name is known — there is no parameter schema, so the tool cannot be invoked. This tool takes a query, matches it against the deferred tool list, and returns the matched tools' complete JSONSchema definitions inside a <functions> block. Once a tool's schema appears in that result, it is callable exactly like any tool defined at the top of this prompt.
+
+Result format: each matched tool appears as one <function>{"description": "...", "name": "...", "parameters": {...}}</function> line inside the <functions> block — the same encoding as the tool list at the top of this prompt.
+
+Query forms:
+- "select:Read,Edit,Grep" — fetch these exact tools by name
+- "notebook jupyter" — keyword search, up to max_results best matches
+- "+slack send" — require "slack" in the name, rank by remaining terms"""
+    input_model = ToolSearchToolInput
+
+    def is_read_only(self, arguments: ToolSearchToolInput) -> bool:
+        del arguments
+        return True
+
+    async def execute(self, arguments: ToolSearchToolInput, context: ToolExecutionContext) -> ToolResult:
+        registry = context.metadata.get("tool_registry") if hasattr(context, "metadata") else None
+        if registry is None:
+            return ToolResult(output="Tool registry context not available", is_error=True)
+
+        query = arguments.query.strip()
+        all_tools = registry.list_tools()
+
+        matches = self._match_tools(query, all_tools)
+
+        if not matches:
+            return ToolResult(output="(no matches)")
+
+        functions_xml = self._build_functions_block(matches)
+        return ToolResult(output=functions_xml)
+
+    def _match_tools(self, query: str, all_tools: list[BaseTool]) -> list[BaseTool]:
+        """根据查询语法匹配工具。
+
+        Args:
+            query: 查询字符串
+            all_tools: 所有已注册工具
+
+        Returns:
+            匹配的工具列表
+        """
+        if query.startswith("select:"):
+            return self._match_select(query, all_tools)
+        if query.startswith("+"):
+            return self._match_require(query, all_tools)
+        return self._match_keyword(query, all_tools)
+
+    def _match_select(self, query: str, all_tools: list[BaseTool]) -> list[BaseTool]:
+        """select:Name1,Name2,... — 按名称精确匹配。"""
+        names = {n.strip() for n in query[len("select:"):].split(",") if n.strip()}
+        return [t for t in all_tools if t.name in names]
+
+    def _match_require(self, query: str, all_tools: list[BaseTool]) -> list[BaseTool]:
+        """+term other... — 名称必须包含 term，按剩余词排名，最多返回 5 个。"""
+        parts = query.split()
+        if not parts:
+            return []
+        required = parts[0][1:]  # 去掉前导 +
+        remaining_terms = parts[1:]
+
+        candidates = [t for t in all_tools if required.lower() in t.name.lower()]
+        if not remaining_terms:
+            return candidates[:5]
+
+        scored = sorted(
+            candidates,
+            key=lambda t: self._keyword_score(t, remaining_terms),
+            reverse=True,
+        )
+        return scored[:5]
+
+    def _match_keyword(self, query: str, all_tools: list[BaseTool]) -> list[BaseTool]:
+        """关键词搜索，按匹配度排序，最多返回 5 个。"""
+        terms = query.lower().split()
+        if not terms:
+            return []
+
+        scored = sorted(
+            all_tools,
+            key=lambda t: self._keyword_score(t, terms),
+            reverse=True,
+        )
+        return [t for t in scored if self._keyword_score(t, terms) > 0][:5]
+
+    @staticmethod
+    def _keyword_score(tool: BaseTool, terms: list[str]) -> int:
+        """计算工具对关键词列表的匹配得分。"""
+        text = (tool.name + " " + tool.description).lower()
+        return sum(1 for term in terms if term in text)
+
+    def _build_functions_block(self, tools: list[BaseTool]) -> str:
+        """将工具列表构建为 <function>JSONSchema</function> 格式。"""
+        lines: list[str] = []
+        for tool in tools:
+            schema_dict = self._tool_to_function_schema(tool)
+            lines.append(f"<function>{json.dumps(schema_dict, ensure_ascii=False)}</function>")
+        return "\n".join(lines)
+
+    @staticmethod
+    def _tool_to_function_schema(tool: BaseTool) -> dict[str, Any]:
+        """将工具转换为 function schema 格式（使用 parameters 键）。"""
+        api_schema = tool.to_api_schema()
+        return {
+            "name": api_schema["name"],
+            "description": api_schema["description"],
+            "parameters": api_schema["input_schema"],
+        }

illusion/tools/web_fetch_tool.py

@@ -0,0 +1,264 @@
+"""
+网页抓取和摘要工具
+==================
+
+本模块提供获取和摘要远程网页内容的功能。
+
+主要组件：
+    - WebFetchTool: 抓取并摘要网页的工具
+
+使用示例：
+    >>> from illusion.tools import WebFetchTool
+    >>> tool = WebFetchTool()
+"""
+
+from __future__ import annotations
+
+import html as _html_module
+import re
+import time
+from urllib.parse import urlparse
+
+import httpx
+from openai import AsyncOpenAI
+from pydantic import BaseModel, Field
+
+from illusion.config.settings import load_settings
+from illusion.tools.base import BaseTool, ToolExecutionContext, ToolResult
+
+# ---------------------------------------------------------------------------
+# 15-minute TTL cache
+# ---------------------------------------------------------------------------
+_cache: dict[str, tuple[float, str]] = {}
+_CACHE_TTL = 15 * 60  # 15 minutes in seconds
+
+
+def _cache_key(url: str, prompt: str, max_chars: int) -> str:
+    return f"{url}|{prompt}|{max_chars}"
+
+
+def _cache_get(key: str) -> str | None:
+    entry = _cache.get(key)
+    if entry is None:
+        return None
+    ts, value = entry
+    if time.time() - ts > _CACHE_TTL:
+        del _cache[key]
+        return None
+    return value
+
+
+def _cache_set(key: str, value: str) -> None:
+    _cache[key] = (time.time(), value)
+
+
+class WebFetchToolInput(BaseModel):
+    """网页抓取参数。
+
+    属性：
+        url: 要抓取的 HTTP 或 HTTPS URL
+        prompt: 描述你想从页面中提取什么信息
+        max_chars: 最大返回字符数（500-50000）
+    """
+
+    url: str = Field(description="HTTP or HTTPS URL to fetch")
+    prompt: str = Field(
+        default="Summarize the key content of this page.",
+        description="Describes what information you want to extract from the page",
+    )
+    max_chars: int = Field(default=12000, ge=500, le=50000)
+
+
+class WebFetchTool(BaseTool):
+    """抓取一个网页并使用 AI 模型处理内容。
+
+    用于获取和分析网络内容。
+    """
+
+    name = "web_fetch"
+    description = """- Fetches content from a specified URL and processes it using an AI model
+- Takes a URL and a prompt as input
+- Fetches the URL content, converts HTML to markdown
+- Processes the content with the prompt using a small, fast model
+- Returns the model's response about the content
+- Use this tool when you need to retrieve and analyze web content
+
+Usage notes:
+  - IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions.
+  - The URL must be a fully-formed valid URL
+  - HTTP URLs will be automatically upgraded to HTTPS
+  - The prompt should describe what information you want to extract from the page
+  - This tool is read-only and does not modify any files
+  - Results may be summarized if the content is very large
+  - Includes a self-cleaning 15-minute cache for faster responses when repeatedly accessing the same URL
+  - When a URL redirects to a different host, the tool will inform you and provide the redirect URL in a special format. You should then make a new WebFetch request with the redirect URL to fetch the content.
+  - For GitHub URLs, prefer using the gh CLI via Bash instead (e.g., gh pr view, gh issue view, gh api)."""
+    input_model = WebFetchToolInput
+
+    async def execute(self, arguments: WebFetchToolInput, context: ToolExecutionContext) -> ToolResult:
+        del context
+        url = arguments.url
+
+        # 自动升级 HTTP 到 HTTPS
+        parsed = urlparse(url)
+        if parsed.scheme == "http":
+            url = url.replace("http://", "https://", 1)
+
+        # 检查缓存
+        ck = _cache_key(url, arguments.prompt, arguments.max_chars)
+        cached = _cache_get(ck)
+        if cached is not None:
+            return ToolResult(output=cached)
+
+        # 发起 HTTP 请求（手动处理重定向以检测跨主机跳转）
+        try:
+            async with httpx.AsyncClient(follow_redirects=False, timeout=20.0) as client:
+                response = await client.get(url, headers={"User-Agent": "IllusionCode/0.1"})
+                # 检测跨主机重定向
+                while response.is_redirect:
+                    location = response.headers.get("location", "")
+                    if not location:
+                        break
+                    redirect_parsed = urlparse(location)
+                    current_parsed = urlparse(url)
+                    # 如果是相对路径或同主机，跟随
+                    if not redirect_parsed.netloc or redirect_parsed.netloc == current_parsed.netloc:
+                        url = location if redirect_parsed.netloc else f"{current_parsed.scheme}://{current_parsed.netloc}{location}"
+                        response = await client.get(url, headers={"User-Agent": "IllusionCode/0.1"})
+                    else:
+                        return ToolResult(
+                            output=(
+                                f"Redirect detected to a different host. The URL {arguments.url} "
+                                f"redirects to:\n\n{location}\n\n"
+                                f"Please make a new WebFetch request with the redirect URL."
+                            )
+                        )
+                response.raise_for_status()
+        except httpx.HTTPError as exc:
+            return ToolResult(output=f"web_fetch failed: {exc}", is_error=True)
+
+        # 处理响应内容
+        content_type = response.headers.get("content-type", "")
+        body = response.text
+        # 如果是 HTML，转换为 Markdown
+        if "html" in content_type:
+            body = _html_to_markdown(body)
+        body = body.strip()
+        # 截断过长的内容
+        if len(body) > arguments.max_chars:
+            body = body[: arguments.max_chars].rstrip() + "\n...[truncated]"
+
+        # 使用 AI 模型处理内容
+        try:
+            ai_response = await _process_with_model(body, arguments.prompt)
+        except Exception:
+            # 模型调用失败时回退到直接返回内容
+            result = (
+                f"URL: {response.url}\n"
+                f"Status: {response.status_code}\n"
+                f"Content-Type: {content_type or '(unknown)'}\n\n"
+                f"{body}"
+            )
+            return ToolResult(output=result)
+
+        _cache_set(ck, ai_response)
+        return ToolResult(output=ai_response)
+
+    def is_read_only(self, arguments: BaseModel) -> bool:
+        del arguments
+        return True
+
+
+async def _process_with_model(content: str, prompt: str) -> str:
+    """使用 AI 模型处理内容。"""
+    settings = load_settings()
+    env = settings._active_env
+    api_key = env.api_key or None
+    base_url = env.base_url or None
+
+    if not api_key:
+        raise RuntimeError("No API key configured")
+
+    if env.api_format == "anthropic":
+        # Anthropic 需要固定 base_url
+        base_url = base_url or "https://api.anthropic.com"
+        client = AsyncOpenAI(api_key=api_key, base_url=f"{base_url}/v1")
+    else:
+        client = AsyncOpenAI(api_key=api_key, base_url=base_url)
+
+    model_name = settings._active_model_name
+
+    system_prompt = (
+        "You are a web content summarizer. Analyze the provided web page content and respond "
+        "to the user's prompt. Be concise and accurate. Only use information from the provided content."
+    )
+
+    resp = await client.chat.completions.create(
+        model=model_name,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": f"Web page content:\n\n{content}\n\nUser prompt: {prompt}"},
+        ],
+        max_tokens=4096,
+        temperature=0.3,
+    )
+    return resp.choices[0].message.content or ""
+
+
+def _html_to_markdown(html_text: str) -> str:
+    """将 HTML 转换为 Markdown。"""
+    text = html_text
+
+    # 移除 script、style、nav、footer、header 标签及其内容
+    text = re.sub(r"(?is)<(script|style|nav|footer|header|noscript).*?>.*?</\1>", " ", text)
+
+    # 标题 h1-h6
+    for i in range(6, 0, -1):
+        text = re.sub(
+            rf"(?is)<h{i}[^>]*>\s*(.*?)\s*</h{i}>",
+            lambda m, n=i: "#" * n + " " + _strip_html(m.group(1)).strip() + "\n\n",
+            text,
+        )
+
+    # 粗体 / 斜体
+    text = re.sub(r"(?is)<(?:b|strong)[^>]*>(.*?)</(?:b|strong)>", r"**\1**", text)
+    text = re.sub(r"(?is)<(?:i|em)[^>]*>(.*?)</(?:i|em)>", r"*\1*", text)
+
+    # 链接（优先处理有 href 的 <a>）
+    text = re.sub(r'(?is)<a[^>]*href=["\']([^"\']+)["\'][^>]*>(.*?)</a>', r"[\2](\1)", text)
+
+    # 图片
+    text = re.sub(r'(?is)<img[^>]*src=["\']([^"\']+)["\'][^>]*/?>', r"![](\1)", text)
+
+    # 段落
+    text = re.sub(r"(?is)<p[^>]*>\s*(.*?)\s*</p>", r"\1\n\n", text)
+
+    # 换行
+    text = re.sub(r"(?is)<br\s*/?>", "\n", text)
+
+    # 无序列表项
+    text = re.sub(r"(?is)<li[^>]*>\s*(.*?)\s*</li>", r"- \1\n", text)
+
+    # 代码块
+    text = re.sub(r"(?is)<pre[^>]*>(.*?)</pre>", r"\n```\n\1\n```\n", text)
+    text = re.sub(r"(?is)<code[^>]*>(.*?)</code>", r"`\1`", text)
+
+    # 删除所有剩余 HTML 标签
+    text = re.sub(r"(?s)<[^>]+>", " ", text)
+
+    # 解码 HTML 实体
+    text = _html_module.unescape(text)
+
+    # 规范化空白和多余空行
+    text = re.sub(r"[ \t\f\r]+", " ", text)
+    text = re.sub(r"\n[ \t]+\n", "\n\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+
+    return text.strip()
+
+
+def _strip_html(fragment: str) -> str:
+    """移除 HTML 标签，保留纯文本。"""
+    text = re.sub(r"(?s)<[^>]+>", " ", fragment)
+    text = _html_module.unescape(text)
+    return re.sub(r"\s+", " ", text).strip()

illusion/tools/web_search_tool.py

@@ -0,0 +1,186 @@
+"""
+简单网页搜索工具
+================
+
+本模块提供执行网络搜索并返回紧凑顶部结果的功能。
+
+主要组件：
+    - WebSearchTool: 执行网页搜索的工具
+
+使用示例：
+    >>> from illusion.tools import WebSearchTool
+    >>> tool = WebSearchTool()
+"""
+
+from __future__ import annotations
+
+import html
+import re
+from urllib.parse import parse_qs, unquote, urlparse
+
+import httpx
+from pydantic import BaseModel, Field
+
+from illusion.tools.base import BaseTool, ToolExecutionContext, ToolResult
+
+
+class WebSearchToolInput(BaseModel):
+    """网页搜索参数。
+
+    属性：
+        query: 搜索查询
+        max_results: 最大结果数量（1-10）
+        search_url: 可选的搜索端点覆盖
+    """
+
+    query: str = Field(description="Search query")
+    max_results: int = Field(default=5, ge=1, le=10, description="Maximum number of results")
+    search_url: str | None = Field(
+        default=None,
+        description="Optional override for the HTML search endpoint, useful for private search backends or testing.",
+    )
+
+
+class WebSearchTool(BaseTool):
+    """运行网络搜索并返回紧凑的顶部结果。
+
+    用于获取超出 Illusion 知识截止日期的最新信息。
+    """
+
+    name = "web_search"
+    description = """- Allows Illusion to search the web and use the results to inform responses
+- Provides up-to-date information for current events and recent data
+- Returns search result information formatted as search result blocks, including links as markdown hyperlinks
+- Use this tool for accessing information beyond Illusion's knowledge cutoff
+- Searches are performed automatically within a single API call
+
+CRITICAL REQUIREMENT - You MUST follow this:
+  - After answering the user's question, you MUST include a "Sources:" section at the end of your response
+  - In the Sources section, list all relevant URLs from the search results as markdown hyperlinks: [Title](URL)
+  - This is MANDATORY - never skip including sources in your response
+  - Example format:
+
+    [Your answer here]
+
+    Sources:
+    - [Source Title 1](https://example.com/1)
+    - [Source Title 2](https://example.com/2)
+
+Usage notes:
+  - Web search is only available in the US
+
+IMPORTANT - Use the correct year in search queries:
+  - The current month is <currentMonthYear>. You MUST use this year when searching for recent information, documentation, or current events.
+  - Example: If the user asks for "latest React docs", search for "React documentation" with the current year, NOT last year"""
+    input_model = WebSearchToolInput
+
+    def is_read_only(self, arguments: WebSearchToolInput) -> bool:
+        del arguments
+        return True
+
+    async def execute(
+        self,
+        arguments: WebSearchToolInput,
+        context: ToolExecutionContext,
+    ) -> ToolResult:
+        del context
+        # 确定搜索端点
+        endpoint = arguments.search_url or "https://html.duckduckgo.com/html/"
+        try:
+            # 发起搜索请求
+            async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
+                response = await client.get(
+                    endpoint,
+                    params={"q": arguments.query},
+                    headers={"User-Agent": "IllusionCode/0.1"},
+                )
+                response.raise_for_status()
+        except httpx.HTTPError as exc:
+            return ToolResult(output=f"web_search failed: {exc}", is_error=True)
+
+        # 解析搜索结果
+        results = _parse_search_results(response.text, limit=arguments.max_results)
+        if not results:
+            return ToolResult(output="No search results found.", is_error=True)
+
+        # 构建输出 — 使用 Markdown 超链接格式
+        lines = [f"Search results for: {arguments.query}"]
+        for index, result in enumerate(results, start=1):
+            lines.append(f"{index}. [{result['title']}]({result['url']})")
+            if result["snippet"]:
+                lines.append(f"   {result['snippet']}")
+        return ToolResult(output="\n".join(lines))
+
+
+def _parse_search_results(body: str, *, limit: int) -> list[dict[str, str]]:
+    """解析搜索结果页面。
+
+    参数：
+        body: 搜索结果页面的 HTML 内容
+        limit: 最大返回结果数
+
+    返回：
+        搜索结果字典列表
+    """
+    # 提取摘要片段
+    snippets = [
+        _clean_html(match.group("snippet"))
+        for match in re.finditer(
+            r'<(?:a|div|span)[^>]+class="[^"]*(?:result__snippet|result-snippet)[^"]*"[^>]*>(?P<snippet>.*?)</(?:a|div|span)>',
+            body,
+            flags=re.IGNORECASE | re.DOTALL,
+        )
+    ]
+
+    results: list[dict[str, str]] = []
+    # 查找所有链接
+    anchor_matches = re.finditer(
+        r"<a(?P<attrs>[^>]+)>(?P<title>.*?)</a>",
+        body,
+        flags=re.IGNORECASE | re.DOTALL,
+    )
+    for index, match in enumerate(anchor_matches):
+        attrs = match.group("attrs")
+        # 检查是否为结果链接
+        class_match = re.search(r'class="(?P<class>[^"]+)"', attrs, flags=re.IGNORECASE)
+        if class_match is None:
+            continue
+        class_names = class_match.group("class")
+        if "result__a" not in class_names and "result-link" not in class_names:
+            continue
+        # 提取 href
+        href_match = re.search(r'href="(?P<href>[^"]+)"', attrs, flags=re.IGNORECASE)
+        if href_match is None:
+            continue
+        # 解析标题和 URL
+        title = _clean_html(match.group("title"))
+        url = _normalize_result_url(href_match.group("href"))
+        snippet = snippets[index] if index < len(snippets) else ""
+        if title and url:
+            results.append({"title": title, "url": url, "snippet": snippet})
+        if len(results) >= limit:
+            break
+    return results
+
+
+def _normalize_result_url(raw_url: str) -> str:
+    """规范化 DuckDuckGo 重定向 URL。
+
+    将 /l/ 路径下的重定向 URL 解析为目标 URL。
+    """
+    parsed = urlparse(raw_url)
+    if parsed.netloc.endswith("duckduckgo.com") and parsed.path.startswith("/l/"):
+        target = parse_qs(parsed.query).get("uddg", [""])[0]
+        return unquote(target) if target else raw_url
+    return raw_url
+
+
+def _clean_html(fragment: str) -> str:
+    """清理 HTML 片段，提取纯文本。"""
+    # 移除 HTML 标签
+    text = re.sub(r"(?s)<[^>]+>", " ", fragment)
+    # 解码 HTML 实体
+    text = html.unescape(text)
+    # 规范化空白
+    text = re.sub(r"\s+", " ", text).strip()
+    return text

illusion/ui/__init__.py

@@ -0,0 +1,23 @@
+"""
+UI 模块
+=======
+
+本模块提供 IllusionCode 用户界面的核心功能。
+
+主要组件：
+    - run_repl: 运行交互式 REPL（默认的 React 终端界面）
+    - run_print_mode: 运行非交互式打印模式（适合脚本和自动化任务）
+
+使用示例：
+    >>> from illusion.ui import run_repl, run_print_mode
+    >>> 
+    >>> # 启动交互式 REPL
+    >>> await run_repl()
+    >>> 
+    >>> # 运行单次交互模式
+    >>> await run_print_mode(prompt="帮我写一个 hello world 程序")
+"""
+
+from illusion.ui.app import run_repl, run_print_mode
+
+__all__ = ["run_repl", "run_print_mode"]