entari-plugin-hyw 4.0.0rc5__py3-none-any.whl → 4.0.0rc7__py3-none-any.whl

entari_plugin_hyw/browser/engines/default.py

@@ -0,0 +1,166 @@
+
+import urllib.parse
+import re
+from typing import List, Dict, Any
+from loguru import logger
+from .base import SearchEngine
+
+
+class DefaultEngine(SearchEngine):
+    """
+    Default browser address bar search engine.
+    Uses the browser's address bar to search (Ctrl+L -> type -> Enter).
+    This uses whatever default search engine the browser is configured with.
+    """
+    
+    # Special marker to indicate this engine uses address bar input
+    USE_ADDRESS_BAR = True
+    
+    def build_url(self, query: str, limit: int = 10) -> str:
+        """
+        For address bar search, we don't build a URL.
+        Return the raw query - SearchService will handle the address bar input.
+        """
+        # Return a special marker so SearchService knows to use address bar
+        return f"__ADDRESS_BAR_SEARCH__:{query}"
+
+    def parse(self, content: str) -> List[Dict[str, Any]]:
+        """
+        Parse search results from whatever search engine the browser uses.
+        We detect the engine from the HTML and use appropriate parsing.
+        """
+        results = []
+        seen_urls = set()
+        
+        # Detect which search engine based on content
+        is_google = 'google' in content.lower() and ('class="g"' in content or 'data-hveid' in content)
+        is_bing = 'bing' in content.lower() and 'b_algo' in content
+        is_duckduckgo = 'duckduckgo' in content.lower()
+        
+        if is_google:
+            results = self._parse_google(content, seen_urls)
+        elif is_bing:
+            results = self._parse_bing(content, seen_urls)
+        elif is_duckduckgo:
+            results = self._parse_duckduckgo(content, seen_urls)
+        else:
+            # Generic fallback
+            results = self._parse_generic(content, seen_urls)
+        
+        logger.info(f"DefaultEngine parsed {len(results)} results (detected: {'google' if is_google else 'bing' if is_bing else 'ddg' if is_duckduckgo else 'generic'})")
+        return results
+    
+    def _parse_google(self, content: str, seen_urls: set) -> List[Dict[str, Any]]:
+        """Parse Google search results."""
+        results = []
+        # Look for result links
+        link_regex = re.compile(
+            r'<a[^>]+href="(https?://(?!google\.com|accounts\.google)[^"]+)"[^>]*>([^<]+)</a>',
+            re.IGNORECASE
+        )
+        
+        for match in link_regex.finditer(content):
+            if len(results) >= 15:
+                break
+            href = match.group(1)
+            title = match.group(2).strip()
+            
+            if href in seen_urls or not title or len(title) < 3:
+                continue
+            if any(x in href for x in ['google.com', 'gstatic.com', 'youtube.com/redirect']):
+                continue
+                
+            seen_urls.add(href)
+            results.append({
+                "title": re.sub(r'<[^>]+>', '', title),
+                "url": href,
+                "domain": urllib.parse.urlparse(href).hostname or "",
+                "content": "",
+            })
+        return results
+    
+    def _parse_bing(self, content: str, seen_urls: set) -> List[Dict[str, Any]]:
+        """Parse Bing search results."""
+        results = []
+        link_regex = re.compile(
+            r'<a[^>]+href="(https?://(?!bing\.com|microsoft\.com)[^"]+)"[^>]*>(.*?)</a>',
+            re.IGNORECASE | re.DOTALL
+        )
+        
+        for match in link_regex.finditer(content):
+            if len(results) >= 15:
+                break
+            href = match.group(1)
+            title_html = match.group(2)
+            title = re.sub(r'<[^>]+>', '', title_html).strip()
+            
+            if href in seen_urls or not title or len(title) < 3:
+                continue
+            if any(x in href for x in ['bing.com', 'microsoft.com', 'msn.com']):
+                continue
+                
+            seen_urls.add(href)
+            results.append({
+                "title": title,
+                "url": href,
+                "domain": urllib.parse.urlparse(href).hostname or "",
+                "content": "",
+            })
+        return results
+    
+    def _parse_duckduckgo(self, content: str, seen_urls: set) -> List[Dict[str, Any]]:
+        """Parse DuckDuckGo results."""
+        results = []
+        link_regex = re.compile(
+            r'<a[^>]+href="(https?://(?!duckduckgo\.com)[^"]+)"[^>]*>(.*?)</a>',
+            re.IGNORECASE | re.DOTALL
+        )
+        
+        for match in link_regex.finditer(content):
+            if len(results) >= 15:
+                break
+            href = match.group(1)
+            title_html = match.group(2)
+            title = re.sub(r'<[^>]+>', '', title_html).strip()
+            
+            if href in seen_urls or not title or len(title) < 3:
+                continue
+                
+            seen_urls.add(href)
+            results.append({
+                "title": title,
+                "url": href,
+                "domain": urllib.parse.urlparse(href).hostname or "",
+                "content": "",
+            })
+        return results
+    
+    def _parse_generic(self, content: str, seen_urls: set) -> List[Dict[str, Any]]:
+        """Generic link parser for unknown search engines."""
+        results = []
+        link_regex = re.compile(
+            r'<a[^>]+href="(https?://[^"]+)"[^>]*>([^<]+)</a>',
+            re.IGNORECASE
+        )
+        
+        for match in link_regex.finditer(content):
+            if len(results) >= 15:
+                break
+            href = match.group(1)
+            title = match.group(2).strip()
+            
+            if href in seen_urls or not title or len(title) < 5:
+                continue
+            # Skip common non-result URLs
+            if any(x in href for x in ['javascript:', 'mailto:', '#', 'login', 'signin', 'account']):
+                continue
+                
+            seen_urls.add(href)
+            results.append({
+                "title": title,
+                "url": href,
+                "domain": urllib.parse.urlparse(href).hostname or "",
+                "content": "",
+            })
+        return results
+

entari_plugin_hyw/browser/engines/{searxng.py → duckduckgo.py}

@@ -5,9 +5,9 @@ from typing import List, Dict, Any
 from loguru import logger
 from .base import SearchEngine
 
-class SearXNGEngine(SearchEngine):
+class DuckDuckGoEngine(SearchEngine):
     """
-    Parser for DuckDuckGo and SearXNG results.
+    Parser for DuckDuckGo Lite results.
     Handles both Markdown (from Crawl4AI) and HTML (fallback).
     """
     
@@ -83,7 +83,7 @@ class SearXNGEngine(SearchEngine):
                 })
                 seen_urls.add(href)
                 
-        logger.info(f"SearXNG Parser(HTML) found {len(results)} results.")
+        logger.info(f"DuckDuckGo Parser(HTML) found {len(results)} results.")
         return results
 
     def _parse_markdown(self, content: str) -> List[Dict[str, Any]]:
@@ -133,5 +133,5 @@ class SearXNGEngine(SearchEngine):
         if current_result:
              results.append(current_result)
         
-        logger.info(f"SearXNG Parser(Markdown) found {len(results)} results.")
+        logger.info(f"DuckDuckGo Parser(Markdown) found {len(results)} results.")
         return results

entari_plugin_hyw/browser/engines/google.py

@@ -0,0 +1,155 @@
+
+import urllib.parse
+import re
+from typing import List, Dict, Any
+from loguru import logger
+from .base import SearchEngine
+
+
+class GoogleEngine(SearchEngine):
+    """
+    Search engine implementation for Google.
+    Parses Google Search HTML results.
+    """
+    
+    def build_url(self, query: str, limit: int = 10) -> str:
+        encoded_query = urllib.parse.quote(query)
+        return f"https://www.google.com/search?q={encoded_query}"
+
+    def parse(self, content: str) -> List[Dict[str, Any]]:
+        results = []
+        seen_urls = set()
+        
+        # Google search results are in blocks with class="MjjYud" or similar containers
+        # Split by result blocks first for more accurate extraction
+        
+        # Method 1: Split by common result block classes
+        block_patterns = [
+            r'<div class="MjjYud"[^>]*>',
+            r'<div class="tF2Cxc"[^>]*>',
+            r'<div class="g Ww4FFb"[^>]*>',
+        ]
+        
+        blocks = [content]
+        for bp in block_patterns:
+            new_blocks = []
+            for block in blocks:
+                parts = re.split(bp, block)
+                new_blocks.extend(parts)
+            blocks = new_blocks
+        
+        for block in blocks:
+            if len(block) < 100:
+                continue
+                
+            # Find URL in this block - prefer links with h3 nearby
+            url_match = re.search(r'<a[^>]+href="(https?://(?!www\.google\.|google\.|webcache\.googleusercontent\.)[^"]+)"[^>]*>', block)
+            if not url_match:
+                continue
+                
+            url = url_match.group(1)
+            if url in seen_urls or self._should_skip_url(url):
+                continue
+            
+            # Find h3 title in this block
+            h3_match = re.search(r'<h3[^>]*>(.*?)</h3>', block, re.IGNORECASE | re.DOTALL)
+            if not h3_match:
+                continue
+                
+            title = re.sub(r'<[^>]+>', '', h3_match.group(1)).strip()
+            if not title or len(title) < 2:
+                continue
+            
+            seen_urls.add(url)
+            
+            # Extract snippet from VwiC3b class (Google's snippet container)
+            snippet = ""
+            snippet_match = re.search(r'<div[^>]*class="[^"]*VwiC3b[^"]*"[^>]*>(.*?)</div>', block, re.IGNORECASE | re.DOTALL)
+            if snippet_match:
+                snippet = re.sub(r'<[^>]+>', ' ', snippet_match.group(1)).strip()
+                snippet = re.sub(r'\s+', ' ', snippet).strip()
+            
+            # Fallback: look for any text after h3
+            if not snippet:
+                # Try other common snippet patterns
+                alt_patterns = [
+                    r'<span[^>]*class="[^"]*aCOpRe[^"]*"[^>]*>(.*?)</span>',
+                    r'<div[^>]*data-snc[^>]*>(.*?)</div>',
+                ]
+                for ap in alt_patterns:
+                    am = re.search(ap, block, re.IGNORECASE | re.DOTALL)
+                    if am:
+                        snippet = re.sub(r'<[^>]+>', ' ', am.group(1)).strip()
+                        snippet = re.sub(r'\s+', ' ', snippet).strip()
+                        break
+            
+            # Extract images from this block
+            images = []
+            # Pattern 1: Regular img src (excluding data: and tracking pixels)
+            # Note: gstatic.com/images/branding is logo, but encrypted-tbn*.gstatic.com are thumbnails
+            img_matches = re.findall(r'<img[^>]+src="(https?://[^"]+)"', block)
+            for img_url in img_matches:
+                # Decode HTML entities
+                img_url = img_url.replace('&amp;', '&')
+                # Skip tracking/icon/small images (but allow encrypted-tbn which are valid thumbnails)
+                if any(x in img_url.lower() for x in ['favicon', 'icon', 'tracking', 'pixel', 'logo', 'gstatic.com/images/branding', '1x1', 'transparent', 'gstatic.com/images/icons']):
+                    continue
+                if img_url not in images:
+                    images.append(img_url)
+            
+            # Pattern 2: data-src (lazy loaded images)
+            data_src_matches = re.findall(r'data-src="(https?://[^"]+)"', block)
+            for img_url in data_src_matches:
+                img_url = img_url.replace('&amp;', '&')
+                if any(x in img_url.lower() for x in ['favicon', 'icon', 'tracking', 'pixel', 'logo']):
+                    continue
+                if img_url not in images:
+                    images.append(img_url)
+            
+            results.append({
+                "title": title,
+                "url": url,
+                "domain": urllib.parse.urlparse(url).hostname or "",
+                "content": snippet[:1000],
+                "images": images[:3]  # Limit to 3 images per result
+            })
+            
+            if len(results) >= 15:
+                break
+        
+        total_images = sum(len(r.get("images", [])) for r in results)
+        logger.info(f"GoogleEngine parsed {len(results)} results with {total_images} images total.")
+        return results
+    
+    def _should_skip_url(self, url: str) -> bool:
+        """Check if URL should be skipped."""
+        skip_patterns = [
+            "google.com",
+            "googleusercontent.com",
+            "gstatic.com",
+            "youtube.com/watch",  # Keep channel/playlist but skip individual videos
+            "maps.google",
+            "translate.google",
+            "accounts.google",
+            "support.google",
+            "policies.google",
+            "schema.org",
+            "javascript:",
+            "data:",
+            "#",
+        ]
+        
+        for pattern in skip_patterns:
+            if pattern in url.lower():
+                return True
+        
+        # Skip very short URLs (likely invalid)
+        if len(url) < 20:
+            return True
+        
+        # Skip URLs that are just root domains without path
+        parsed = urllib.parse.urlparse(url)
+        if not parsed.path or parsed.path == "/":
+            return True
+            
+        return False

entari_plugin_hyw/browser/manager.py

@@ -65,7 +65,7 @@ class SharedBrowserManager:
             # Hide scrollbars globally
             co.set_argument('--hide-scrollbars')
             # 十万的原因是滚动条屏蔽(大概吧)
-            co.set_argument('--window-size=1280,20000')
+            co.set_argument('--window-size=1280,9000')
             self._page = ChromiumPage(addr_or_opts=co)
             
             # Show Landing Page