entari-plugin-hyw 3.2.112__py3-none-any.whl → 3.3.0__py3-none-any.whl

entari_plugin_hyw/utils/prompts.py

@@ -1,92 +1,128 @@
-VISION_SYSTEM_PROMPT = """你是一个专业的视觉转文字专家.
+VISION_SP = """# 你是一个专业的视觉转文字专家.
 
-[用户消息]
-{user_msgs}
-
-[核心任务]
+# 核心任务
 - 智能分析图片内容, 转述成文本, 除此之外不要添加任何内容
 - 文字优先: 若包含清晰文字（文档、截图等）, 必须完整准确转录, 不要遗漏.
 - 视觉补充: 若无文字, 重点描述视觉内容（物体、场景、氛围）.
 - 用户要求: 根据用户消息中提示侧重转文本的偏向, 若无或无关联则不理会常规完成.
-"""
-
-INTRUCT_SYSTEM_PROMPT = """你是一个专业的指导专家.
 
-[用户消息]
+## 用户消息
+```text
 {user_msgs}
+```
+"""
 
-[核心任务]
-- 决定是否使用搜索工具
-  - 如果用户消息包含典型名词、可能的专有名词组合, 且意图为解释此词, 请使用搜索工具, 搜索工具会给你返回最新的资料和图片.
-  - 如果用户消息明显不需要搜索, 或虽然存在名词但是作为过程参与不涉及结果, 则不调用搜索工具
-  - 如果用户的消息明显有两个搜索的方向, 本次对话最多同时调用两个搜索工具分开搜索
-  - 理解用户话语, 提炼出搜索关键词.
-    - 保持原意, 禁止添加额外内容.
-    - 禁止擅自分割关键词导致语意变化.
-- 决定是否放权 mcp工具 给 agent
-  - 如果用户显式地表达了要求模型使用mcp帮助完成任务的意图, 调用工具放权
+INTRUCT_SP = """# 你是一个专业的指导专家.
+
+## 核心任务
+- 决定预处理工具:
+  - 用户消息包含链接: 调用 crawl_page 获取内容, 无需其他工具
+  - 用户消息包含典型名词、可能的专有名词组合: internal_web_search (提炼出关键词搜索, 保持原意, 指向不同的领域时优先搜索最贴切的, 最多同时调用2)
+  - 用户消息适合加入图片点缀: internal_image_search
+  - 用户消息不需要搜索: 不调用工具
+- 调用 set_mode:
+  - 绝大部分常规问题: standard
+  - 用户要求研究/深度搜索: agent
+  - 需要获取页面具体信息才能回答问题: agent
 > 所有工具需要在本次对话同时调用
 
-[调用工具]
+## 调用工具
+- 使用工具时, 必须通过 function_call / tool_call 机制调用.
 {tools_desc}
+
+## 你的回复
+调用工具后无需额外文本.
+
+## 用户消息
+```
+{user_msgs}
+```
 """
 
-INTRUCT_SYSTEM_PROMPT_VISION_ADD = """
-[视觉专家消息]
+
+INTRUCT_SP_VISION_ADD = """
+## 视觉专家消息
+```text
 {vision_msgs}
+```
 """
 
+AGENT_SP = """# 你是一个 Agent 总控专家, 你需要理解用户意图, 根据已有信息给出最终回复.
+> 请确保你输出的任何消息有着准确的来源, 减少输出错误信息.
 
-AGENT_SYSTEM_PROMPT = """
-你是一个全能助手, 请根据用户需求和搜索结果中贴切用户意图的可靠信息解释用户消息中的关键词.
+当前模式: {mode}, {mode_desc}
 
-[用户消息]
-{user_msgs}
+## 最终回复格式要求
+- 直接输出 Markdown 正文.
 
-[回复格式要求]
 当不调用工具发送文本, 即会变成最终回复, 请遵守: 
-- 语言: 简体中文, 百科式风格.
-- 正文格式: 使用 Markdown, 有大标题, 可以使用数学公式, 格式内容丰富.
+- 语言: 简体中文, 百科式风格, 语言严谨不啰嗦.
+- 正文格式: 使用 Markdown格式, [hightlight, katex], 有大标题, 内容丰富突出重点.
+- 工具引用: 
+  - 搜索摘要引用: 使用 `search:数字id` 如 `search:3`
+  - 页面内容引用: 使用 `page:数字id` 如 `page:5`
+  - 每个引用必须分开标注
+  - 在正文底部添加 references 代码块:
+    - 用不到的条目不写, 没有专家给信息就不写.
+    ```references
+    [1] [search] [文本描述](url)
+    [3] [search] [文本描述](url)
+    [5] [page] [页面标题](url)
+    [7] [page] [页面标题](url)
+    ```
+
+## 用户消息
+```text
+{user_msgs}
+```
 """
 
-AGENT_SYSTEM_PROMPT_INTRUCT_VISION_ADD = """
-[视觉专家消息]
-{vision_msgs}
+# PS: agent 无搜索图片权限
+AGENT_SP_TOOLS_STANDARD_ADD = """
+你需要整合已有的信息, 提炼用户消息中的关键词, 进行最终回复.
 """
 
-AGENT_SYSTEM_PROMPT_SEARCH_ADD = """
-[搜索专家给出的信息]
-{search_msgs}
 
-[最终回复]
-- 图片: 如果本次回答适合配图, 对搜索到的图片, 选择 1-3 张合适的尽量类型、来源、不同、主题契合的图片, 美观分布嵌入正文 `![alt](url)`.
-- 搜索引用: 在正文中使用 `ref:数字id` (代码形式) 如 `ref:1` 标注来源, 并在文末通过 `references` 代码块列出, 每个引用必须分开标注.
-- 你需要在最终回复底部添加 `references` 代码块.
+AGENT_SP_TOOLS_AGENT_ADD = """
+- 你现在可以使用工具: {tools_desc}
+  - 你需要判断顺序或并发使用工具获取信息:
+    - 0-1 次 internal_web_search
+    - 0-1 次 internal_image_search (如果用户需要图片, 通常和 internal_web_search 并发执行)
+    - 1-2 次 crawl_page
+- 使用工具时, 必须通过 function_call / tool_call 机制调用.
+"""
+
 
-```references
-1. [标题](url)
-2. [标题](url)
+
+AGENT_SP_INTRUCT_VISION_ADD = """
+## 视觉专家消息
+```text
+{vision_msgs}
 ```
 """
 
-AGENT_SYSTEM_PROMPT_MCP_ADD = """
-[MCP 工具已授权]
-可用工具:
-{tools_desc}
-
-> 积极使用工具完成任务，工具优先于文本回复。
-秧根扭起来，我们喜羊羊大东北是我的家乡，我就在这一个人
-[最终回复格式]
-- 工具引用: 在正文中使用 `mcp:字母顺序` (代码形式) 如 `mcp:a` 标注来源, 并在文末通过 `mcp` 代码块列出, 每个引用必须分开标注.
-- 底部添加 `mcp` 代码块列出工具调用流程:
-- 格式: `[图标] 工具名称` + 文本描述
-- 图标: navigate, snapshot, click, type, code, wait, default
-
-```mcp
-1. [code] browser_run_code: 执行JavaScript计算
-2. [navigate] navigate: 导航到xxx网站
+AGENT_SP_SEARCH_ADD = """
+## 搜索专家消息
+```text
+{search_msgs}
 ```
+
+
 """
 
+AGENT_SP_PAGE_ADD = """
+## 页面内容专家消息
+```text
+{page_msgs}
+```
+- 引用页面内容时, 必须使用 `page:id` 格式
+"""
 
+AGENT_SP_IMAGE_SEARCH_ADD = """
+## 图像搜索专家消息
+```text
+{image_search_msgs}
+```
+- 每进行一次 internal_image_search, 挑选 1 张图像插入正文
+"""
 

entari_plugin_hyw/utils/search.py

@@ -1,193 +1,241 @@
-import re
-import httpx
 import urllib.parse
 from typing import List, Dict, Optional, Any
 from loguru import logger
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.cache_context import CacheMode
+
+# Shared crawler instance to avoid repeated init
+_shared_crawler: Optional[AsyncWebCrawler] = None
+
+
+async def get_shared_crawler() -> AsyncWebCrawler:
+    global _shared_crawler
+    if _shared_crawler is None:
+        _shared_crawler = AsyncWebCrawler()
+        await _shared_crawler.start()
+    return _shared_crawler
+
+
+async def close_shared_crawler():
+    global _shared_crawler
+    if _shared_crawler:
+        try:
+            await _shared_crawler.close()
+        except Exception:
+            pass
+        _shared_crawler = None
 
 class SearchService:
     """
-    Specialized service for interacting with SearXNG.
-    Uses regex-based HTML parsing to ensure O(n) performance and zero blocking,
-    bypasssing heavy DOM parsers like Trafilatura.
+    Crawl4AI-backed search & fetch service.
+    Uses the configured search engine results page (SERP) URL and parses links from the HTML.
     """
     def __init__(self, config: Any):
         self.config = config
+        self._default_limit = 8
+        self._crawler: Optional[AsyncWebCrawler] = None
 
-    async def search(self, query: str) -> List[Dict[str, str]]:
-        """
-        Execute search and parse results using Regex.
-        Returns a list of dicts: {'title': str, 'url': str, 'content': str}
-        """
-        # 1. Construct URL (Force HTML format since JSON is 403)
+    def _build_search_url(self, query: str) -> str:
         encoded_query = urllib.parse.quote(query)
-        base = getattr(self.config, "search_base_url", "http://127.0.0.1:8888/search?")
-        
-        # Ensure we don't have double '?' or '&' issues
+        base = getattr(self.config, "search_base_url", "https://lite.duckduckgo.com/lite/?q={query}")
+        if "{query}" in base:
+            return base.replace("{query}", encoded_query).replace("{limit}", str(self._default_limit))
         sep = "&" if "?" in base else "?"
-        
-        # Remove any existing format=json if present in base (just in case)
-        base = base.replace("format=json&", "").replace("&format=json", "")
-        
-        # Handle {query} placeholder if present (common in config defaults)
+        return f"{base}{sep}q={encoded_query}"
+
+    def _build_image_url(self, query: str) -> str:
+        encoded_query = urllib.parse.quote(query)
+        base = getattr(self.config, "image_search_base_url", "https://duckduckgo.com/?q={query}&iax=images&ia=images")
         if "{query}" in base:
-            # We need to handle potential other placeholders like {limit} if they exist, or escape them
-            # For simplicity, we just replace {query} and ignore format/limit changes since we parse HTML
-            # Actually, standard python format() might fail if other braces exist.
-            # safe replace:
-            url = base.replace("{query}", encoded_query)
-            # Remove other common placeholders if they linger
-            url = url.replace("{limit}", "8")
-        else:
-            # Append mode
-            url = f"{base}{sep}q={encoded_query}&language=zh-CN"
-        
-        logger.info(f"SearchService: Fetching {url}")
+            return base.replace("{query}", encoded_query).replace("{limit}", str(self._default_limit))
+        sep = "&" if "?" in base else "?"
+        return f"{base}{sep}q={encoded_query}&iax=images&ia=images"
+
+    async def search(self, query: str) -> List[Dict[str, str]]:
+        """
+        Crawl the configured SERP using Crawl4AI and return parsed results.
+        """
+        if not query:
+            return []
+
+        url = self._build_search_url(query)
+        logger.info(f"SearchService(Crawl4AI): fetching {url}")
 
         try:
-            async with httpx.AsyncClient(timeout=10.0) as client:
-                resp = await client.get(url)
-                if resp.status_code != 200:
-                    logger.error(f"Search failed: {resp.status_code}")
-                    return []
-                html = resp.text
-                return self._parse_searxng_html(html)
+            crawler = await self._get_crawler()
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    wait_until="domcontentloaded",
+                    wait_for="article",
+                    cache_mode=CacheMode.BYPASS,
+                    word_count_threshold=1,
+                    screenshot=False,
+                    capture_console_messages=False,
+                    capture_network_requests=False,
+                ),
+            )
+            return self._parse_markdown_result(result, limit=self._default_limit)
         except Exception as e:
-            logger.error(f"Search execution failed: {e}")
+            logger.error(f"Crawl4AI search failed: {e}")
             return []
 
-    def _parse_searxng_html(self, html: str) -> List[Dict[str, str]]:
+    def _parse_markdown_result(self, result, limit: int = 8) -> List[Dict[str, str]]:
+        """Parse Crawl4AI result into search items without manual HTML parsing."""
+        md = (result.markdown or result.extracted_content or "").strip()
+        lines = [ln.strip() for ln in md.splitlines() if ln.strip()]
+        links = result.links.get("external", []) if getattr(result, "links", None) else []
+        seen = set()
+        results: List[Dict[str, str]] = []
+
+        def find_snippet(url: str, domain: str) -> str:
+            for ln in lines:
+                if url in ln or (domain and domain in ln):
+                    return ln[:400]
+            # fallback to first non-empty line
+            return lines[0][:400] if lines else ""
+
+        for link in links:
+            url = link.get("href") or ""
+            if not url or url in seen:
+                continue
+            seen.add(url)
+            domain = urllib.parse.urlparse(url).hostname or ""
+            title = link.get("title") or link.get("text") or url
+            snippet = find_snippet(url, domain)
+            results.append({
+                "title": title.strip(),
+                "url": url,
+                "domain": domain,
+                "content": snippet or title,
+            })
+            if len(results) >= limit:
+                break
+
+        if not results:
+            logger.warning(f"SearchService: no results parsed; md_length={len(md)}, links={len(links)}")
+        else:
+            logger.info(f"SearchService: parsed {len(results)} results via Crawl4AI links")
+        return results
+
+    async def fetch_page(self, url: str) -> Dict[str, str]:
         """
-        Parse SearXNG HTML results using Regex.
-        Target structure:
-        <article class="result ...">
-           <h3><a href="(url)">(title)</a></h3>
-           <p class="content">(snippet)</p>
-        </article>
+        Fetch a single page via Crawl4AI and return cleaned markdown/text plus metadata.
         """
-        results = []
-        
-        # Regex to find result blocks. 
-        # We split by <article to find chunks, then parse each chunk.
-        # This is safer than a global regex which might get confused by nested structures.
-        chunks = html.split('<article')
-        
-        for chunk in chunks[1:]: # Skip preamble
-            try:
-                # 1. Extract URL and Title
-                # Look for <a href="..." ... >Title</a> inside h3
-                # Simplified pattern: href="([^"]+)" text is >([^<]+)<
-                link_match = re.search(r'href="([^"]+)".*?>([^<]+)<', chunk)
-                if not link_match:
-                    continue
-                
-                url = link_match.group(1)
-                title = link_match.group(2).strip()
-                
-                # Verify it's a valid result link (sometimes engine links appear)
-                if "searxng" in url or url.startswith("/"):
-                    continue
+        if not url:
+            return {"content": "Error: missing url", "title": "Error", "url": ""}
 
-                # 2. Extract Snippet
-                # Look for class="content">...<
-                # We try to capture text until the next tag open
-                snippet_match = re.search(r'class="content"[^>]*>([\s\S]*?)</p>', chunk)
-                snippet = ""
-                if snippet_match:
-                    # Clean up HTML tags from snippet if any remain (basic check)
-                    raw_snippet = snippet_match.group(1)
-                    snippet = re.sub(r'<[^>]+>', '', raw_snippet).strip()
-                
-                if url and title:
-                    # SAFETY: Truncate snippet to 500 chars to prevent context explosion
-                    final_snippet = (snippet or title)[:500]
-                    results.append({
-                        "title": title,
-                        "url": url,
-                        "content": final_snippet
-                    })
-                    
-                if len(results) >= 8: # Limit to 8 results
-                    break
-                    
+        try:
+            crawler = await self._get_crawler()
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    wait_until="networkidle",
+                    wait_for_images=False,  # Faster: skip image loading
+                    cache_mode=CacheMode.BYPASS,
+                    word_count_threshold=1,
+                    screenshot=False,
+                    capture_console_messages=False,
+                    capture_network_requests=False,
+                ),
+            )
+            if not result.success:
+                return {"content": f"Error: crawl failed ({result.error_message or 'unknown'})", "title": "Error", "url": url}
+            
+            content = result.markdown or result.extracted_content or result.cleaned_html or result.html or ""
+            # Extract metadata if available, otherwise fallback
+            title = "No Title"
+            if result.metadata:
+                title = result.metadata.get("title") or result.metadata.get("og:title") or title
+            
+            # If metadata title is missing/generic, try to grab from links or url? No, metadata is best.
+            if title == "No Title" and result.links:
+                 # Minimal fallback not really possible without parsing HTML again or regex
+                 pass
+
+            return {
+                "content": content[:8000], 
+                "title": title, 
+                "url": result.url or url 
+            }
+        except Exception as e:
+            logger.error(f"Crawl4AI fetch failed: {e}")
+            return {"content": f"Error: crawl failed ({e})", "title": "Error", "url": url}
+
+    async def _get_crawler(self) -> AsyncWebCrawler:
+        # Prefer shared crawler to minimize INIT logs; fall back to local if needed
+        try:
+            return await get_shared_crawler()
+        except Exception as e:
+            logger.warning(f"Shared crawler unavailable, creating local: {e}")
+            if self._crawler is None:
+                self._crawler = AsyncWebCrawler()
+                await self._crawler.start()
+            return self._crawler
+
+    async def close(self):
+        if self._crawler:
+            try:
+                await self._crawler.close()
             except Exception:
-                continue
-                
-        logger.info(f"SearchService: Parsed {len(results)} results")
-        return results
+                pass
+            self._crawler = None
 
     async def image_search(self, query: str) -> List[Dict[str, str]]:
         """
-        Perform image search using regex parsing on HTML results.
+        Image search via Crawl4AI media extraction.
         """
-        if not query: return []
-        
-        encoded_query = urllib.parse.quote(query)
-        base = getattr(self.config, "image_search_base_url", "http://127.0.0.1:8888/search?")
-        sep = "&" if "?" in base else "?"
-        
-        # Clean format=json
-        base = base.replace("format=json&", "").replace("&format=json", "")
-        
-        if "{query}" in base:
-            url = base.replace("{query}", encoded_query)
-            url = url.replace("{limit}", "8")
-        else:
-            url = f"{base}{sep}q={encoded_query}&iax=images&ia=images"
-        
-        logger.info(f"SearchService: Fetching Images {url}")
-        
+        if not query:
+            return []
+
+        url = self._build_image_url(query)
+        logger.info(f"SearchService(Crawl4AI Image): fetching {url}")
+
         try:
-            async with httpx.AsyncClient(timeout=10.0) as client:
-                resp = await client.get(url)
-                resp.raise_for_status()
-                html_content = resp.text
+            # Use image crawler (text_mode=False) for image search
+            crawler = await self._get_crawler()
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    wait_until="networkidle",
+                    wait_for_images=True,
+                    wait_for="img",
+                    cache_mode=CacheMode.BYPASS,
+                    word_count_threshold=1,
+                    screenshot=False,
+                    capture_console_messages=False,
+                    capture_network_requests=False,
+                ),
+            )
+            images = []
+            seen = set()
+            for img in result.media.get("images", []):
+                src = img.get("src") or ""
+                if not src:
+                    continue
+                if src.startswith("//"):
+                    src = "https:" + src
+                if not src.startswith("http"):
+                    continue
+                if src in seen:
+                    continue
+                seen.add(src)
+                alt = (img.get("alt") or img.get("desc") or "").strip()
+                domain = urllib.parse.urlparse(src).hostname or ""
+                images.append({
+                    "title": alt or "Image",
+                    "url": src,
+                    "domain": domain,
+                    "content": alt or "Image",
+                })
+                if len(images) >= self._default_limit:
+                    break
+            if not images:
+                logger.warning(f"SearchService: no images parsed; media_count={len(result.media.get('images', []))}")
+            else:
+                logger.info(f"SearchService: parsed {len(images)} images via Crawl4AI media")
+            return images
         except Exception as e:
-            logger.error(f"Image Search failed: {e}")
+            logger.error(f"Crawl4AI image search failed: {e}")
             return []
-
-        # Regex for Images (DuckDuckGo style / Generic)
-        # DDG images usually in a script or complex layout. 
-        # For simplicity in V2 regex approach, we look for common img tags with logical classes or structure
-        # OR, since the user's SearXNG likely returns standard HTML list for images too.
-        # SearXNG Image results usually: <img src="..." alt="..."> inside a result container.
-        # Let's try a generic pattern for SearXNG image results
-        
-        results = []
-        # SearXNG pattern: <div class="img-search-result"> ... <img src="URL" ...>
-        # Or just look for img tags with src that are http
-        
-        # More robust SearXNG specific regex:
-        # Pattern: <img class="image" src="(?P<url>[^"]+)" alt="(?P<title>[^"]+)"
-        # This is a guess. Let's try to match standard "result_image" or similar if possible.
-        
-        # Assuming SearXNG:
-        # More robust regex to capture images from various engines (SearXNG, Google, Bing)
-        # 1. Try generic <img ... src="..."> with http
-        # 2. Try to extract alt text if available
-        
-        # Pattern 1: Standard img tag with src
-        # We look for src="http..." and optional alt
-        image_matches = re.finditer(r'<img[^>]+src=["\'](http[^"\']+)["\'][^>]*>', html_content, re.IGNORECASE)
-        
-        for match in image_matches:
-            img_tag = match.group(0)
-            img_url = match.group(1)
-            
-            # Extract alt/title
-            alt_match = re.search(r'alt=["\']([^"\']*)["\']', img_tag, re.IGNORECASE)
-            title = alt_match.group(1) if alt_match else ""
-            
-            # Filter out tiny icons/favicons/data uris if possible
-            if "favicon" in img_url or "static" in img_url or "data:image" in img_url:
-                continue
-                
-            results.append({
-                "title": title or "Image",
-                "url": img_url,
-                "content": f"Image: {title}"
-            })
-            
-            if len(results) >= 8:
-                break
-                
-        return results

{entari_plugin_hyw-3.2.112.dist-info → entari_plugin_hyw-3.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: entari_plugin_hyw
-Version: 3.2.112
+Version: 3.3.0
 Summary: Use large language models to interpret chat messages
 Author-email: kumoSleeping <zjr2992@outlook.com>
 License: MIT
@@ -18,15 +18,10 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: arclet-entari[full]>=0.16.5
 Requires-Dist: openai
-Requires-Dist: mcp
 Requires-Dist: httpx
 Requires-Dist: markdown>=3.10
-Requires-Dist: trafilatura>=2.0.0
-Requires-Dist: playwright>=1.56.0
+Requires-Dist: crawl4ai>=0.7.8
 Requires-Dist: jinja2>=3.0
-Provides-Extra: playwright
-Requires-Dist: playwright>=1.56.0; extra == "playwright"
-Requires-Dist: trafilatura>=2.0.0; extra == "playwright"
 Provides-Extra: dev
 Requires-Dist: entari-plugin-server>=0.5.0; extra == "dev"
 Requires-Dist: satori-python-adapter-onebot11>=0.2.5; extra == "dev"
@@ -43,7 +38,9 @@ Requires-Dist: satori-python-adapter-onebot11>=0.2.5; extra == "dev"
 
 </div>
 
-# v3.2迎来大幅度改动、现在图文不符
+# v3.3 迎来大幅度改动、现在图文不符
+
+
 
 ## 🎑 效果展示
 
@@ -140,3 +137,6 @@ hyw -t 一大段话。
 ### 引用回复
 支持引用消息进行追问，机器人会自动读取被引用的消息作为上下文：
 - **引用 + 命令**：机器人将理解被引用消息的内容（包括图片）通过 `MessageChain` 操作拼接 `Text`、`Image` 与部分 `Custom`。
+
+UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. 
+GitHub. https://github.com/unclecode/crawl4ai