jarvis-ai-assistant 0.1.130__py3-none-any.whl → 0.1.132__py3-none-any.whl

jarvis/jarvis_tools/read_webpage.py

@@ -1,7 +1,8 @@
 from typing import Dict, Any
-import requests
-from bs4 import BeautifulSoup
-from yaspin import yaspin
+from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+from bs4 import BeautifulSoup, Tag
+from urllib.parse import urlparse, urljoin
+import re
 
 from jarvis.jarvis_utils.output import OutputType, PrettyOutput
 
@@ -20,93 +21,206 @@ class WebpageTool:
     }
 
     def execute(self, args: Dict) -> Dict[str, Any]:
-        """Read webpage content"""
+        """Read webpage content using Playwright to handle JavaScript-rendered pages"""
         try:
-            url = args["url"].strip()   
+            url = args["url"].strip()
             
-            # Set request headers
-            headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-            }
-            
-            # Send request
-            with yaspin(text="正在读取网页...", color="cyan") as spinner:
-                response = requests.get(url, headers=headers, timeout=10)
-                response.raise_for_status()
-                spinner.text = "网页读取完成"
-                spinner.ok("✅")
+            with sync_playwright() as p:
+                # Launch browser
+                browser = p.chromium.launch(
+                    headless=True,
+                    args=['--disable-gpu', '--no-sandbox', '--disable-dev-shm-usage']
+                )
                 
-            
-            # Use correct encoding
-
-            response.encoding = response.apparent_encoding
-            
-            # Parse HTML
-            with yaspin(text="正在解析网页...", color="cyan") as spinner:
-                soup = BeautifulSoup(response.text, 'html.parser')
-                spinner.text = "网页解析完成"
-                spinner.ok("✅")
-            
-            # Remove script and style tags
-            with yaspin(text="正在移除脚本和样式...", color="cyan") as spinner:
-                for script in soup(["script", "style"]):
-                    script.decompose()
-                spinner.text = "脚本和样式移除完成"
-                spinner.ok("✅")
-            
-            # Extract title
-            with yaspin(text="正在提取标题...", color="cyan") as spinner:
-                title = soup.title.string if soup.title else ""
-                title = title.strip() if title else "No title"
-                spinner.text = "标题提取完成"
-                spinner.ok("✅")
-            
-            with yaspin(text="正在提取文本和链接...", color="cyan") as spinner:
-                # Extract text and links
-                text_parts = []
-                links = []
+                # Create a new page with appropriate settings
+                page = browser.new_page(
+                    user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+                    viewport={'width': 1920, 'height': 1080}
+                )
                 
-                # Process content and collect links
-                for element in soup.descendants:
-                    if element.name == 'a' and element.get('href'): # type: ignore
-                        href = element.get('href') # type: ignore
-                        text = element.get_text(strip=True)
-                        if text and href:
-                            links.append(f"[{text}]({href})")
-                    elif isinstance(element, str) and element.strip():
-                        text_parts.append(element.strip())
-                spinner.text = "文本和链接提取完成"
-                spinner.ok("✅")
-            
-            # Build output
-            with yaspin(text="正在构建输出...", color="cyan") as spinner:
+                # Set timeout to avoid long waits
+                page.set_default_timeout(30000)  # 30 seconds
+                
+                try:
+                    # Navigate to URL and wait for page to load
+                    response = page.goto(url, wait_until="domcontentloaded")
+                    
+                    # Additional wait for network to be idle (with a timeout)
+                    try:
+                        page.wait_for_load_state("networkidle", timeout=10000)
+                    except PlaywrightTimeoutError:
+                        # Continue even if network doesn't become completely idle
+                        pass
+                        
+                    # Make sure we got a valid response
+                    if not response or response.status >= 400:
+                        raise Exception(f"Failed to load page: HTTP {response.status if response else 'No response'}")
+                    
+                    # Get page title safely
+                    title = "No title"
+                    try:
+                        title = page.title()
+                    except Exception:
+                        # Try to extract title from content if direct method fails
+                        try:
+                            title_element = page.query_selector("title")
+                            if title_element:
+                                title = title_element.text_content() or "No title"
+                        except Exception:
+                            pass
+                    
+                    # Get the HTML content after JavaScript execution
+                    html_content = page.content()
+                    
+                except Exception as e:
+                    raise Exception(f"Error navigating to page: {str(e)}")
+                finally:
+                    # Always close browser
+                    browser.close()
+                
+                # Parse with BeautifulSoup and convert to markdown
+                markdown_content = self._html_to_markdown(html_content, url)
+                
+                # Build output in markdown format
                 output = [
-                    f"Title: {title}",
-                    "",
-                    "Text content:",
-                    "\n".join(text_parts),
-                    "",
-                    "Links found:",
-                    "\n".join(links) if links else "No links found"
+                    f"# {title}",
+                    f"Url: {url}",
+                    markdown_content
                 ]
-                spinner.text = "输出构建完成"
-                spinner.ok("✅")
-            
-            return {
-                "success": True,
-                "stdout": "\n".join(output),
-                "stderr": ""
-            }
+                
+                return {
+                    "success": True,
+                    "stdout": "\n".join(output),
+                    "stderr": ""
+                }
 
-        except requests.RequestException as e:
-            return {
-                "success": False,
-                "stdout": "",
-                "stderr": f"Webpage request failed: {str(e)}"
-            }
         except Exception as e:
+            PrettyOutput.print(f"读取网页失败: {str(e)}", OutputType.ERROR)
             return {
                 "success": False,
                 "stdout": "",
                 "stderr": f"Failed to parse webpage: {str(e)}"
-            } 
+            }
+    
+    def _create_soup_element(self, content):
+        """Safely create a BeautifulSoup element, ensuring it's treated as markup"""
+        if isinstance(content, str):
+            # Create a wrapper tag to ensure proper parsing
+            soup_div = BeautifulSoup(f"<div>{content}</div>", 'html.parser').div
+            if soup_div is not None:
+                return soup_div.contents
+            # Return an empty list if the div is None
+            return []
+        return content
+    
+    def _html_to_markdown(self, html_content: str, base_url: str) -> str:
+        """Convert HTML to Markdown format preserving the content structure"""
+        soup = BeautifulSoup(html_content, 'html.parser')
+        
+        # Remove unwanted elements
+        for element in soup(['script', 'style', 'meta', 'noscript', 'head']):
+            element.decompose()
+        
+        # Process headings
+        for level in range(1, 7):
+            for heading in soup.find_all(f'h{level}'):
+                text = heading.get_text().strip()
+                heading_md = "\n\n" + "#" * level + " " + text + "\n\n"
+                new_element = self._create_soup_element(heading_md)
+                heading.replace_with(*new_element)
+        
+        # Process paragraphs
+        for p in soup.find_all('p'):
+            text = p.get_text().strip()
+            if text:
+                new_element = self._create_soup_element("\n\n" + text + "\n\n")
+                p.replace_with(*new_element)
+        
+        # Process unordered lists
+        for ul in soup.find_all('ul'):
+            items = []
+            for li in ul.find_all('li', recursive=False):
+                items.append("* " + li.get_text().strip())
+            new_element = self._create_soup_element("\n\n" + "\n".join(items) + "\n\n")
+            ul.replace_with(*new_element)
+        
+        # Process ordered lists
+        for ol in soup.find_all('ol'):
+            items = []
+            for i, li in enumerate(ol.find_all('li', recursive=False), 1):
+                items.append(str(i) + ". " + li.get_text().strip())
+            new_element = self._create_soup_element("\n\n" + "\n".join(items) + "\n\n")
+            ol.replace_with(*new_element)
+        
+        # Process links (first pass)
+        for a in soup.find_all('a', href=True):
+            try:
+                href = a['href']
+                text = a.get_text().strip()
+                if text and href:
+                    # Convert relative URLs to absolute
+                    if href.startswith('/') and not href.startswith('//'):
+                        href = urljoin(base_url, href)
+                    link_md = "[" + text + "](" + href + ")"
+                    new_element = self._create_soup_element(link_md)
+                    a.replace_with(*new_element)
+            except (KeyError, AttributeError):
+                continue
+        
+        # Process images
+        for img in soup.find_all('img', src=True):
+            try:
+                src = img['src']
+                alt = img.get('alt', 'Image').strip()
+                # Convert relative URLs to absolute
+                if src.startswith('/') and not src.startswith('//'):
+                    src = urljoin(base_url, src)
+                img_md = "![" + alt + "](" + src + ")"
+                new_element = self._create_soup_element(img_md)
+                img.replace_with(*new_element)
+            except (KeyError, AttributeError, UnboundLocalError):
+                continue
+        
+        # Process code blocks
+        for pre in soup.find_all('pre'):
+            code = pre.get_text().strip()
+            pre_md = "\n\n```\n" + code + "\n```\n\n"
+            new_element = self._create_soup_element(pre_md)
+            pre.replace_with(*new_element)
+        
+        # Process inline code
+        for code in soup.find_all('code'):
+            text = code.get_text().strip()
+            code_md = "`" + text + "`"
+            new_element = self._create_soup_element(code_md)
+            code.replace_with(*new_element)
+        
+        # Process line breaks
+        for br in soup.find_all('br'):
+            new_element = self._create_soup_element('\n')
+            br.replace_with(*new_element)
+        
+        # Get the full text
+        markdown_text = soup.get_text()
+        
+        # Clean up extra whitespace and line breaks
+        markdown_text = re.sub(r'\n{3,}', '\n\n', markdown_text)
+        markdown_text = re.sub(r'\s{2,}', ' ', markdown_text)
+        
+        # Process links again (for any that might have been missed)
+        link_pattern = r'\[([^\]]+)\]\(([^)]+)\)'
+        all_links = re.findall(link_pattern, markdown_text)
+        
+        # Add a section with all links at the end
+        if all_links:
+            link_section = ["", "## Links", ""]
+            seen_links = set()
+            for text, href in all_links:
+                link_entry = "[" + text + "](" + href + ")"
+                if link_entry not in seen_links:
+                    link_section.append(link_entry)
+                    seen_links.add(link_entry)
+            
+            markdown_text += "\n\n" + "\n".join(link_section)
+        
+        return markdown_text.strip() 

jarvis/jarvis_tools/registry.py

@@ -13,20 +13,21 @@ from jarvis.jarvis_tools.base import Tool
 from jarvis.jarvis_utils.config import get_max_token_count
 from jarvis.jarvis_utils.embedding import get_context_token_count
 from jarvis.jarvis_utils.output import OutputType, PrettyOutput
+from jarvis.jarvis_utils.utils import ct, ot, init_env
 
 
 
-tool_call_help = """
+tool_call_help = f"""
 # 🛠️ 工具使用系统
 您正在使用一个需要精确格式和严格规则的工具执行系统。
 
 # 📋 工具调用格式
-<TOOL_CALL>
+{ot("TOOL_CALL")}
 name: 工具名称
 arguments:
     param1: 值1
     param2: 值2
-</TOOL_CALL>
+{ct("TOOL_CALL")}
 
 # ❗ 关键规则
 1. 每次只使用一个工具
@@ -53,12 +54,12 @@ arguments:
 # 📝 字符串参数格式
 始终使用 | 语法表示字符串参数：
 
-<TOOL_CALL>
+{ot("TOOL_CALL")}
 name: execute_shell
 arguments:
     command: |
         git status --porcelain
-</TOOL_CALL>
+{ct("TOOL_CALL")}
 
 # 💡 最佳实践
 - 准备好后立即开始执行
@@ -233,7 +234,7 @@ class ToolRegistry(OutputHandler):
             Exception: 如果工具调用缺少必要字段
         """
         # 将内容拆分为行
-        data = re.findall(r'<TOOL_CALL>(.*?)</TOOL_CALL>', content, re.DOTALL)
+        data = re.findall(ot("TOOL_CALL")+r'(.*?)'+ct("TOOL_CALL"), content, re.DOTALL)
         ret = []
         for item in data:
             try:
@@ -270,18 +271,18 @@ class ToolRegistry(OutputHandler):
             name = tool_call["name"]
             args = tool_call["arguments"]
 
-            tool_call_help = """
+            tool_call_help = f"""
 # 🛠️ 工具使用系统
 您正在使用一个需要精确格式和严格规则的工具执行系统。
 
 # 📋 工具调用格式
 
-<TOOL_CALL>
+{ot("TOOL_CALL")}
 name: 工具名称
 arguments:
     param1: 值1
     param2: 值2
-</TOOL_CALL>
+{ct("TOOL_CALL")}
 
 # ❗ 关键规则
 1. 每次只使用一个工具
@@ -308,12 +309,12 @@ arguments:
 # 📝 字符串参数格式
 始终使用 | 语法表示字符串参数：
 
-<TOOL_CALL>
+{ot("TOOL_CALL")}
 name: execute_shell
 arguments:
     command: |
         git status --porcelain
-</TOOL_CALL>
+{ct("TOOL_CALL")}
 
 # 💡 最佳实践
 - 准备好后立即开始执行
@@ -394,3 +395,123 @@ arguments:
         except Exception as e:
             PrettyOutput.print(f"工具执行失败：{str(e)}", OutputType.ERROR)
             return f"工具调用失败: {str(e)}"
+
+
+def main():
+    """命令行工具入口，提供工具列表查看和工具调用功能"""
+    import argparse
+    import json
+
+    init_env()
+
+    parser = argparse.ArgumentParser(description='Jarvis 工具系统命令行界面')
+    subparsers = parser.add_subparsers(dest='command', help='命令')
+
+    # 列出工具子命令
+    list_parser = subparsers.add_parser('list', help='列出所有可用工具')
+    list_parser.add_argument('--json', action='store_true', help='以JSON格式输出')
+    list_parser.add_argument('--detailed', action='store_true', help='显示详细信息')
+
+    # 调用工具子命令
+    call_parser = subparsers.add_parser('call', help='调用指定工具')
+    call_parser.add_argument('tool_name', help='要调用的工具名称')
+    call_parser.add_argument('--args', type=str, help='工具参数 (JSON格式)')
+    call_parser.add_argument('--args-file', type=str, help='从文件加载工具参数 (JSON格式)')
+
+    args = parser.parse_args()
+    
+    # 初始化工具注册表
+    registry = ToolRegistry()
+    
+    if args.command == 'list':
+        tools = registry.get_all_tools()
+        
+        if args.json:
+            if args.detailed:
+                print(json.dumps(tools, indent=2, ensure_ascii=False))
+            else:
+                simple_tools = [{"name": t["name"], "description": t["description"]} for t in tools]
+                print(json.dumps(simple_tools, indent=2, ensure_ascii=False))
+        else:
+            PrettyOutput.section("可用工具列表", OutputType.SYSTEM)
+            for tool in tools:
+                print(f"\n✅ {tool['name']}")
+                print(f"   描述: {tool['description']}")
+                if args.detailed:
+                    print(f"   参数:")
+                    params = tool['parameters'].get('properties', {})
+                    required = tool['parameters'].get('required', [])
+                    for param_name, param_info in params.items():
+                        req_mark = "*" if param_name in required else ""
+                        desc = param_info.get('description', '无描述')
+                        print(f"     - {param_name}{req_mark}: {desc}")
+    
+    elif args.command == 'call':
+        tool_name = args.tool_name
+        tool = registry.get_tool(tool_name)
+        
+        if not tool:
+            PrettyOutput.print(f"错误: 工具 '{tool_name}' 不存在", OutputType.ERROR)
+            available_tools = ", ".join([t["name"] for t in registry.get_all_tools()])
+            print(f"可用工具: {available_tools}")
+            return 1
+        
+        # 获取参数
+        tool_args = {}
+        if args.args:
+            try:
+                tool_args = json.loads(args.args)
+            except json.JSONDecodeError:
+                PrettyOutput.print("错误: 参数必须是有效的JSON格式", OutputType.ERROR)
+                return 1
+        
+        elif args.args_file:
+            try:
+                with open(args.args_file, 'r', encoding='utf-8') as f:
+                    tool_args = json.load(f)
+            except (json.JSONDecodeError, FileNotFoundError) as e:
+                PrettyOutput.print(f"错误: 无法从文件加载参数: {str(e)}", OutputType.ERROR)
+                return 1
+        
+        # 检查必需参数
+        required_params = tool.parameters.get('required', [])
+        missing_params = [p for p in required_params if p not in tool_args]
+        
+        if missing_params:
+            PrettyOutput.print(f"错误: 缺少必需参数: {', '.join(missing_params)}", OutputType.ERROR)
+            print("\n参数说明:")
+            params = tool.parameters.get('properties', {})
+            for param_name in required_params:
+                param_info = params.get(param_name, {})
+                desc = param_info.get('description', '无描述')
+                print(f"  - {param_name}: {desc}")
+            return 1
+        
+        # 执行工具
+        with yaspin(text=f"正在执行工具 {tool_name}...").dots12:
+            result = registry.execute_tool(tool_name, tool_args)
+        
+        # 显示结果
+        if result["success"]:
+            PrettyOutput.section(f"工具 {tool_name} 执行成功", OutputType.SUCCESS)
+        else:
+            PrettyOutput.section(f"工具 {tool_name} 执行失败", OutputType.ERROR)
+            
+        if result.get("stdout"):
+            print("\n输出:")
+            print(result["stdout"])
+            
+        if result.get("stderr"):
+            PrettyOutput.print("\n错误:", OutputType.ERROR)
+            print(result["stderr"])
+            
+        return 0 if result["success"] else 1
+    
+    else:
+        parser.print_help()
+    
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())