auto-coder 1.0.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

autocoder/rag/tools/metaso_example.py

@@ -0,0 +1,268 @@
+
+"""
+Metaso 集成使用示例
+
+该文件展示如何使用集成了 Metaso 的 web_search_tool 和 web_crawl_tool。
+"""
+
+import os
+import sys
+from typing import Dict, Any
+
+# 添加项目路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
+
+from autocoder.rag.tools.metaso_sdk import MetasoClient
+from autocoder.rag.tools.web_search_tool import WebSearchTool, WebSearchToolResolver
+from autocoder.rag.tools.web_crawl_tool import WebCrawlTool, WebCrawlToolResolver
+from unittest.mock import Mock
+
+
+def example_direct_metaso_sdk():
+    """直接使用 Metaso SDK 的示例"""
+    print("=" * 60)
+    print("示例 1: 直接使用 Metaso SDK")
+    print("=" * 60)
+    
+    # 从环境变量获取 API key
+    api_key = os.getenv('METASO_API_KEY')
+    if not api_key:
+        print("\n⚠️  请先设置环境变量 METASO_API_KEY")
+        print("   export METASO_API_KEY='your-api-key-here'")
+        return
+    
+    # 创建客户端
+    client = MetasoClient(api_key=api_key)
+    
+    # 1. 搜索示例
+    print("\n1. 搜索示例:")
+    search_result = client.search(
+        query="Python 编程最佳实践",
+        size=3,
+        include_summary=False
+    )
+    
+    if search_result.success:
+        print(f"   找到 {len(search_result.webpages)} 个结果:")
+        for idx, page in enumerate(search_result.webpages, 1):
+            print(f"\n   {idx}. {page.title}")
+            print(f"      URL: {page.link}")
+            print(f"      摘要: {page.snippet[:100]}...")
+            if page.date:
+                print(f"      日期: {page.date}")
+    else:
+        print(f"   搜索失败: {search_result.error}")
+    
+    # 2. 读取网页示例
+    print("\n2. 读取网页示例:")
+    if search_result.success and search_result.webpages:
+        first_url = search_result.webpages[0].link
+        print(f"   读取 URL: {first_url}")
+        
+        content = client.read(first_url)
+        if not content.startswith("Error:"):
+            print(f"   成功读取，内容长度: {len(content)} 字符")
+            print(f"   内容预览:\n{content[:500]}...")
+        else:
+            print(f"   读取失败: {content}")
+
+
+def example_web_search_tool():
+    """使用 WebSearchTool 的示例"""
+    print("\n" + "=" * 60)
+    print("示例 2: 使用 WebSearchTool (集成 Metaso)")
+    print("=" * 60)
+    
+    # 创建 mock agent 和 args
+    mock_agent = Mock()
+    mock_args = Mock()
+    
+    # 设置 API keys (可以同时设置，系统会自动选择)
+    mock_args.metaso_api_key = os.getenv('METASO_API_KEY')
+    mock_args.firecrawl_api_key = os.getenv('FIRECRAWL_API_KEY')
+    
+    if not mock_args.metaso_api_key:
+        print("\n⚠️  请先设置环境变量 METASO_API_KEY")
+        return
+    
+    # 创建搜索工具 - 明确指定使用 Metaso
+    tool = WebSearchTool(
+        query="人工智能在医疗领域的应用",
+        limit=5,
+        provider="metaso",  # 明确指定使用 Metaso
+        scrape_options='{"include_summary": true}'
+    )
+    
+    # 创建解析器并执行
+    resolver = WebSearchToolResolver(mock_agent, tool, mock_args)
+    result = resolver.resolve()
+    
+    if result.success:
+        print(f"\n搜索成功: {result.message}")
+        print(f"找到 {len(result.content)} 个结果:")
+        
+        for idx, item in enumerate(result.content, 1):
+            print(f"\n{idx}. {item['title']}")
+            print(f"   URL: {item['url']}")
+            print(f"   描述: {item['description'][:100]}...")
+            print(f"   来源: {item.get('provider', 'unknown')}")
+            if 'date' in item:
+                print(f"   日期: {item['date']}")
+    else:
+        print(f"\n搜索失败: {result.message}")
+
+
+def example_web_crawl_tool():
+    """使用 WebCrawlTool 的示例"""
+    print("\n" + "=" * 60)
+    print("示例 3: 使用 WebCrawlTool (集成 Metaso)")
+    print("=" * 60)
+    
+    # 创建 mock agent 和 args
+    mock_agent = Mock()
+    mock_args = Mock()
+    mock_args.metaso_api_key = os.getenv('METASO_API_KEY')
+    mock_args.firecrawl_api_key = os.getenv('FIRECRAWL_API_KEY')
+    
+    if not mock_args.metaso_api_key:
+        print("\n⚠️  请先设置环境变量 METASO_API_KEY")
+        return
+    
+    # 示例 1: 单页读取
+    print("\n1. 单页读取:")
+    tool = WebCrawlTool(
+        url="https://www.python.org",
+        limit=1,
+        provider="metaso"
+    )
+    
+    resolver = WebCrawlToolResolver(mock_agent, tool, mock_args)
+    result = resolver.resolve()
+    
+    if result.success:
+        print(f"   {result.message}")
+        if result.content:
+            page = result.content[0]
+            print(f"   URL: {page['url']}")
+            print(f"   内容长度: {len(page['content'])} 字符")
+            print(f"   内容预览: {page['content'][:200]}...")
+    else:
+        print(f"   失败: {result.message}")
+    
+    # 示例 2: 多页爬取
+    print("\n2. 多页爬取:")
+    tool = WebCrawlTool(
+        url="https://docs.python.org",
+        limit=3,
+        include_paths="/tutorial",
+        provider="metaso"
+    )
+    
+    resolver = WebCrawlToolResolver(mock_agent, tool, mock_args)
+    result = resolver.resolve()
+    
+    if result.success:
+        print(f"   {result.message}")
+        for idx, page in enumerate(result.content, 1):
+            print(f"\n   页面 {idx}:")
+            print(f"     URL: {page['url']}")
+            print(f"     标题: {page.get('title', 'N/A')}")
+            print(f"     内容长度: {len(page.get('content', ''))} 字符")
+    else:
+        print(f"   失败: {result.message}")
+
+
+def example_auto_provider_selection():
+    """演示提供商自动选择"""
+    print("\n" + "=" * 60)
+    print("示例 4: 提供商自动选择")
+    print("=" * 60)
+    
+    mock_agent = Mock()
+    mock_args = Mock()
+    
+    # 设置环境变量中的 API keys
+    mock_args.metaso_api_key = os.getenv('METASO_API_KEY')
+    mock_args.firecrawl_api_key = os.getenv('FIRECRAWL_API_KEY')
+    
+    print("\n环境配置:")
+    print(f"  Metaso API Key: {'已设置' if mock_args.metaso_api_key else '未设置'}")
+    print(f"  Firecrawl API Key: {'已设置' if mock_args.firecrawl_api_key else '未设置'}")
+    
+    if not (mock_args.metaso_api_key or mock_args.firecrawl_api_key):
+        print("\n⚠️  请至少设置一个 API key")
+        return
+    
+    # 不指定 provider，让系统自动选择
+    tool = WebSearchTool(
+        query="自动选择测试",
+        limit=2
+        # 注意：没有指定 provider 参数
+    )
+    
+    resolver = WebSearchToolResolver(mock_agent, tool, mock_args)
+    
+    # 系统会根据可用的 API keys 自动选择：
+    # 1. 如果只有 Metaso key -> 使用 Metaso
+    # 2. 如果只有 Firecrawl key -> 使用 Firecrawl  
+    # 3. 如果两个都有 -> 默认使用 Firecrawl
+    
+    result = resolver.resolve()
+    
+    if result.success:
+        print(f"\n自动选择结果: {result.message}")
+        if "Metaso" in result.message:
+            print("  ✓ 系统选择了 Metaso")
+        elif "Firecrawl" in result.message:
+            print("  ✓ 系统选择了 Firecrawl")
+    else:
+        print(f"\n执行失败: {result.message}")
+
+
+def main():
+    """运行所有示例"""
+    print("\n" + "=" * 60)
+    print("Metaso 集成使用示例")
+    print("=" * 60)
+    
+    print("\n使用说明:")
+    print("1. 设置 API Key:")
+    print("   export METASO_API_KEY='your-metaso-api-key'")
+    print("   export FIRECRAWL_API_KEY='your-firecrawl-api-key' (可选)")
+    print("\n2. 在工具中指定 provider 参数:")
+    print("   provider='metaso' - 使用 Metaso")
+    print("   provider='firecrawl' - 使用 Firecrawl")
+    print("   provider=None - 自动选择")
+    
+    # 检查环境变量
+    has_metaso = bool(os.getenv('METASO_API_KEY'))
+    has_firecrawl = bool(os.getenv('FIRECRAWL_API_KEY'))
+    
+    print("\n当前环境:")
+    print(f"  Metaso API Key: {'✓ 已设置' if has_metaso else '✗ 未设置'}")
+    print(f"  Firecrawl API Key: {'✓ 已设置' if has_firecrawl else '✗ 未设置'}")
+    
+    if not has_metaso:
+        print("\n⚠️  请先设置 METASO_API_KEY 环境变量来运行示例")
+        print("   export METASO_API_KEY='mk-YOUR_API_KEY_HERE'")
+        return
+    
+    try:
+        # 运行示例
+        example_direct_metaso_sdk()
+        example_web_search_tool()
+        example_web_crawl_tool()
+        example_auto_provider_selection()
+        
+        print("\n" + "=" * 60)
+        print("✅ 所有示例运行完成！")
+        print("=" * 60)
+        
+    except Exception as e:
+        print(f"\n❌ 示例运行失败: {str(e)}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

autocoder/rag/tools/metaso_sdk.py

@@ -0,0 +1,417 @@
+"""
+Metaso SDK 封装
+
+该模块提供了 Metaso API 的 Python SDK 封装，支持网页搜索和内容读取功能。
+"""
+
+import json
+import time
+from typing import Dict, Any, List, Optional, Union
+from dataclasses import dataclass
+import requests
+from loguru import logger
+
+
+@dataclass
+class MetasoSearchResult:
+    """Metaso 搜索结果数据类"""
+    title: str
+    link: str
+    snippet: str
+    score: str = "medium"
+    position: int = 0
+    date: Optional[str] = None
+    authors: Optional[List[str]] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典格式"""
+        result = {
+            "title": self.title,
+            "link": self.link,
+            "snippet": self.snippet,
+            "score": self.score,
+            "position": self.position
+        }
+        if self.date:
+            result["date"] = self.date
+        if self.authors:
+            result["authors"] = self.authors
+        return result
+
+
+@dataclass
+class MetasoSearchResponse:
+    """Metaso 搜索响应数据类"""
+    credits: int
+    search_parameters: Dict[str, Any]
+    webpages: List[MetasoSearchResult]
+    success: bool = True
+    error: Optional[str] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典格式"""
+        return {
+            "credits": self.credits,
+            "searchParameters": self.search_parameters,
+            "webpages": [page.to_dict() for page in self.webpages],
+            "success": self.success,
+            "error": self.error
+        }
+
+
+class MetasoClient:
+    """Metaso API 客户端"""
+    
+    BASE_URL = "https://metaso.cn/api/v1"
+    DEFAULT_TIMEOUT = 30  # 默认超时时间（秒）
+    
+    def __init__(self, api_key: str, timeout: int = DEFAULT_TIMEOUT):
+        """
+        初始化 Metaso 客户端
+        
+        Args:
+            api_key: Metaso API 密钥
+            timeout: 请求超时时间（秒）
+        """
+        self.api_key = api_key
+        self.timeout = timeout
+        self.session = requests.Session()
+        self.session.headers.update({
+            "Authorization": f"Bearer {api_key}",
+            "Accept": "application/json",
+            "Content-Type": "application/json"
+        })
+    
+    def search(
+        self,
+        query: str,
+        scope: str = "webpage",
+        size: Union[int, str] = 10,
+        include_summary: bool = False,
+        include_raw_content: bool = False,
+        concise_snippet: bool = False,
+        format: str = "chat_completions"
+    ) -> MetasoSearchResponse:
+        """
+        执行网页搜索
+        
+        Args:
+            query: 搜索查询字符串
+            scope: 搜索范围，如 "webpage"、"news" 等
+            size: 返回结果数量
+            include_summary: 是否包含摘要
+            include_raw_content: 是否包含原始内容
+            concise_snippet: 是否使用简洁摘要
+            format: 返回格式类型
+            
+        Returns:
+            MetasoSearchResponse: 搜索响应对象
+        """
+        try:
+            # 准备请求数据
+            data = {
+                "q": query,
+                "scope": scope,
+                "size": str(size),  # 确保是字符串
+                "includeSummary": include_summary,
+                "includeRawContent": include_raw_content,
+                "conciseSnippet": concise_snippet
+            }
+            
+            # 如果不是默认格式，添加format参数
+            if format != "chat_completions":
+                data["format"] = format
+            
+            logger.info(f"Metaso 搜索请求: query={query}, scope={scope}, size={size}")
+            
+            # 发送请求
+            response = self.session.post(
+                f"{self.BASE_URL}/search",
+                json=data,
+                timeout=self.timeout
+            )
+            
+            # 检查响应状态
+            if response.status_code != 200:
+                error_msg = f"API 请求失败，状态码: {response.status_code}"
+                try:
+                    error_data = response.json()
+                    if "error" in error_data:
+                        error_msg = f"{error_msg}, 错误信息: {error_data['error']}"
+                except:
+                    error_msg = f"{error_msg}, 响应内容: {response.text}"
+                
+                logger.error(error_msg)
+                return MetasoSearchResponse(
+                    credits=0,
+                    search_parameters={},
+                    webpages=[],
+                    success=False,
+                    error=error_msg
+                )
+            
+            # 解析响应
+            result = response.json()
+            
+            # 解析搜索结果
+            webpages = []
+            for idx, item in enumerate(result.get("webpages", [])):
+                webpage = MetasoSearchResult(
+                    title=item.get("title", ""),
+                    link=item.get("link", ""),
+                    snippet=item.get("snippet", ""),
+                    score=item.get("score", "medium"),
+                    position=item.get("position", idx + 1),
+                    date=item.get("date"),
+                    authors=item.get("authors")
+                )
+                webpages.append(webpage)
+            
+            return MetasoSearchResponse(
+                credits=result.get("credits", 0),
+                search_parameters=result.get("searchParameters", {}),
+                webpages=webpages,
+                success=True
+            )
+            
+        except requests.exceptions.Timeout:
+            error_msg = f"搜索请求超时（{self.timeout}秒）"
+            logger.error(error_msg)
+            return MetasoSearchResponse(
+                credits=0,
+                search_parameters={},
+                webpages=[],
+                success=False,
+                error=error_msg
+            )
+        except requests.exceptions.RequestException as e:
+            error_msg = f"网络请求失败: {str(e)}"
+            logger.error(error_msg)
+            return MetasoSearchResponse(
+                credits=0,
+                search_parameters={},
+                webpages=[],
+                success=False,
+                error=error_msg
+            )
+        except Exception as e:
+            error_msg = f"搜索失败: {str(e)}"
+            logger.error(error_msg)
+            return MetasoSearchResponse(
+                credits=0,
+                search_parameters={},
+                webpages=[],
+                success=False,
+                error=error_msg
+            )
+    
+    def read(
+        self,
+        url: str,
+        format: str = "text/plain"
+    ) -> Union[str, Dict[str, Any]]:
+        """
+        读取网页内容
+        
+        Args:
+            url: 要读取的网页 URL
+            format: 返回格式，"text/plain" 或 "application/json"
+            
+        Returns:
+            Union[str, Dict]: 网页内容，根据format参数返回文本或JSON
+        """
+        try:
+            # 准备请求头
+            headers = self.session.headers.copy()
+            headers["Accept"] = format
+            
+            # 准备请求数据
+            data = {
+                "url": url
+            }
+            
+            logger.info(f"Metaso 读取请求: url={url}, format={format}")
+            
+            # 发送请求
+            response = self.session.post(
+                f"{self.BASE_URL}/reader",
+                json=data,
+                headers=headers,
+                timeout=self.timeout
+            )
+            
+            # 检查响应状态
+            if response.status_code != 200:
+                error_msg = f"API 请求失败，状态码: {response.status_code}"
+                try:
+                    error_data = response.json()
+                    if "error" in error_data:
+                        error_msg = f"{error_msg}, 错误信息: {error_data['error']}"
+                except:
+                    error_msg = f"{error_msg}, 响应内容: {response.text}"
+                
+                logger.error(error_msg)
+                if format == "application/json":
+                    return {"success": False, "error": error_msg, "content": ""}
+                else:
+                    return f"Error: {error_msg}"
+            
+            # 根据格式返回结果
+            if format == "application/json":
+                return response.json()
+            else:
+                return response.text
+            
+        except requests.exceptions.Timeout:
+            error_msg = f"读取请求超时（{self.timeout}秒）"
+            logger.error(error_msg)
+            if format == "application/json":
+                return {"success": False, "error": error_msg, "content": ""}
+            else:
+                return f"Error: {error_msg}"
+        except requests.exceptions.RequestException as e:
+            error_msg = f"网络请求失败: {str(e)}"
+            logger.error(error_msg)
+            if format == "application/json":
+                return {"success": False, "error": error_msg, "content": ""}
+            else:
+                return f"Error: {error_msg}"
+        except Exception as e:
+            error_msg = f"读取失败: {str(e)}"
+            logger.error(error_msg)
+            if format == "application/json":
+                return {"success": False, "error": error_msg, "content": ""}
+            else:
+                return f"Error: {error_msg}"
+    
+    def crawl(
+        self,
+        url: str,
+        limit: int = 10,
+        max_depth: Optional[int] = None,
+        exclude_paths: Optional[List[str]] = None,
+        include_paths: Optional[List[str]] = None,
+        allow_subdomains: bool = False
+    ) -> List[Dict[str, Any]]:
+        """
+        爬取网站内容（使用搜索和读取组合实现）
+        
+        Args:
+            url: 起始 URL
+            limit: 最大爬取页面数
+            max_depth: 最大爬取深度
+            exclude_paths: 排除的路径列表
+            include_paths: 包含的路径列表
+            allow_subdomains: 是否允许子域名
+            
+        Returns:
+            List[Dict]: 爬取的页面内容列表
+        """
+        try:
+            from urllib.parse import urlparse, urljoin
+            
+            # 解析起始 URL
+            parsed_url = urlparse(url)
+            domain = parsed_url.netloc
+            
+            # 首先搜索该域名下的相关页面
+            search_query = f"site:{domain}"
+            if parsed_url.path and parsed_url.path != "/":
+                # 如果有特定路径，添加到搜索中
+                search_query += f" {parsed_url.path}"
+            
+            logger.info(f"开始爬取: {url}, 限制: {limit} 页")
+            
+            # 执行搜索获取相关页面
+            search_result = self.search(
+                query=search_query,
+                size=limit,
+                include_raw_content=True
+            )
+            
+            if not search_result.success:
+                logger.error(f"爬取失败: {search_result.error}")
+                return []
+            
+            crawled_pages = []
+            urls_to_crawl = [url]  # 起始URL
+            
+            # 添加搜索结果中的URL
+            for webpage in search_result.webpages:
+                if webpage.link and webpage.link not in urls_to_crawl:
+                    # 检查是否应该包含这个URL
+                    should_include = True
+                    
+                    # 检查排除路径
+                    if exclude_paths:
+                        for exclude_path in exclude_paths:
+                            if exclude_path in webpage.link:
+                                should_include = False
+                                break
+                    
+                    # 检查包含路径
+                    if include_paths and should_include:
+                        should_include = False
+                        for include_path in include_paths:
+                            if include_path in webpage.link:
+                                should_include = True
+                                break
+                    
+                    # 检查子域名
+                    if not allow_subdomains and should_include:
+                        webpage_domain = urlparse(webpage.link).netloc
+                        if webpage_domain != domain:
+                            should_include = False
+                    
+                    if should_include:
+                        urls_to_crawl.append(webpage.link)
+            
+            # 限制爬取数量
+            urls_to_crawl = urls_to_crawl[:limit]
+            
+            # 读取每个页面的内容
+            for idx, page_url in enumerate(urls_to_crawl):
+                logger.info(f"读取页面 {idx + 1}/{len(urls_to_crawl)}: {page_url}")
+                
+                content = self.read(page_url, format="text/plain")
+                
+                # 构建页面结果
+                page_result = {
+                    "url": page_url,
+                    "title": "",  # 从内容中提取标题
+                    "content": content if not content.startswith("Error:") else "",
+                    "markdown": content if not content.startswith("Error:") else "",
+                    "links": [],
+                    "metadata": {
+                        "position": idx + 1,
+                        "source": "metaso"
+                    }
+                }
+                
+                # 尝试从搜索结果中获取标题
+                for webpage in search_result.webpages:
+                    if webpage.link == page_url:
+                        page_result["title"] = webpage.title
+                        page_result["metadata"]["date"] = webpage.date
+                        page_result["metadata"]["authors"] = webpage.authors
+                        break
+                
+                crawled_pages.append(page_result)
+                
+                # 添加小延迟避免请求过快
+                if idx < len(urls_to_crawl) - 1:
+                    time.sleep(0.5)
+            
+            return crawled_pages
+            
+        except Exception as e:
+            logger.error(f"爬取失败: {str(e)}")
+            return []
+    
+    def __enter__(self):
+        """上下文管理器入口"""
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """上下文管理器退出，关闭会话"""
+        self.session.close()