entari-plugin-hyw 4.0.0rc4__py3-none-any.whl → 4.0.0rc6__py3-none-any.whl

entari_plugin_hyw/browser/manager.py

@@ -0,0 +1,153 @@
+"""
+Shared Browser Manager (DrissionPage)
+
+Manages a single ChromiumPage browser instance.
+Supports multi-tab concurrency via DrissionPage's built-in tab management.
+"""
+
+import threading
+from typing import Optional, Any
+from loguru import logger
+from DrissionPage import ChromiumPage, ChromiumOptions
+from DrissionPage.errors import PageDisconnectedError
+
+class SharedBrowserManager:
+    """
+    Manages a shared DrissionPage Chromium browser.
+    """
+    
+    _instance: Optional["SharedBrowserManager"] = None
+    _lock = threading.Lock()
+    
+    def __init__(self, headless: bool = True):
+        self.headless = headless
+        self._page: Optional[ChromiumPage] = None
+        self._starting = False
+        self._tab_lock = threading.Lock()
+    
+    @classmethod
+    def get_instance(cls, headless: bool = True) -> "SharedBrowserManager":
+        """Get or create the singleton SharedBrowserManager."""
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls(headless=headless)
+            return cls._instance
+    
+    def start(self) -> bool:
+        """Start the Chromium browser."""
+        if self._page is not None:
+            # Check if alive
+            try:
+                if self._page.run_cdp('Browser.getVersion'):
+                    return True
+            except (PageDisconnectedError, Exception):
+                self._page = None
+        
+        with self._lock:
+            if self._starting:
+                return False
+            self._starting = True
+        
+        try:
+            logger.info("SharedBrowserManager: Starting DrissionPage browser...")
+            
+            # Configure options
+            co = ChromiumOptions()
+            co.headless(self.headless)
+            co.auto_port()  # Auto find available port
+            
+            # Anti-detection settings 
+            co.set_argument('--no-sandbox')
+            co.set_argument('--disable-gpu')
+            # Essential for loading local files and avoiding CORS issues
+            co.set_argument('--allow-file-access-from-files')
+            co.set_argument('--disable-web-security')
+            # Hide scrollbars globally
+            co.set_argument('--hide-scrollbars')
+            # 十万的原因是滚动条屏蔽(大概吧)
+            co.set_argument('--window-size=1280,20000')
+            self._page = ChromiumPage(addr_or_opts=co)
+            
+            # Show Landing Page
+            try:
+                import os
+                landing_path = os.path.join(os.path.dirname(__file__), 'landing.html')
+                if os.path.exists(landing_path):
+                    self._page.get(f"file://{landing_path}")
+            except Exception as e:
+                logger.warning(f"SharedBrowserManager: Failed to show landing page: {e}")
+
+            logger.success(f"SharedBrowserManager: Browser ready (port={self._page.address})")
+            return True
+            
+        except Exception as e:
+            logger.error(f"SharedBrowserManager: Failed to start browser: {e}")
+            self._page = None
+            raise
+        finally:
+            self._starting = False
+    
+    @property
+    def page(self) -> Optional[ChromiumPage]:
+        """Get the main ChromiumPage instance."""
+        if self._page is None:
+            self.start()
+        return self._page
+
+    def new_tab(self, url: str) -> Any:
+        """Thread-safe tab creation."""
+        with self._tab_lock:
+            page = self.page
+            if not page:
+                raise RuntimeError("Browser not available")
+            return page.new_tab(url)
+    
+    def close(self):
+        """Shutdown the browser."""
+        with self._lock:
+            if self._page:
+                try:
+                    self._page.quit()
+                    logger.info("SharedBrowserManager: Browser closed.")
+                except Exception as e:
+                    logger.warning(f"SharedBrowserManager: Error closing browser: {e}")
+                finally:
+                    self._page = None
+
+    @staticmethod
+    def hide_scrollbars(page: ChromiumPage):
+        """
+        Robustly hide scrollbars using CDP commands.
+        This eliminates the reserved space/gutter that standard CSS might miss.
+        """
+        try:
+            # Emulation.setScrollbarsHidden is a CDP command
+            page.run_cdp('Emulation.setScrollbarsHidden', hidden=True)
+            logger.debug("SharedBrowserManager: CDP scrollbars hidden.")
+        except Exception as e:
+            logger.warning(f"SharedBrowserManager: Failed to hide scrollbars via CDP: {e}")
+
+
+# Module-level singleton accessor
+_shared_manager: Optional[SharedBrowserManager] = None
+
+
+def get_shared_browser_manager(headless: bool = True) -> SharedBrowserManager:
+    """
+    Get or create the shared browser manager.
+    """
+    global _shared_manager
+    
+    if _shared_manager is None:
+        _shared_manager = SharedBrowserManager.get_instance(headless=headless)
+        _shared_manager.start()
+    
+    return _shared_manager
+
+
+def close_shared_browser():
+    """Close the shared browser manager."""
+    global _shared_manager
+    if _shared_manager:
+        _shared_manager.close()
+        _shared_manager = None

entari_plugin_hyw/browser/service.py

@@ -0,0 +1,304 @@
+"""
+Browser Service (DrissionPage)
+
+Provides page fetching and screenshot capabilities using DrissionPage.
+"""
+
+import asyncio
+import base64
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, Dict, Any, List
+from loguru import logger
+import trafilatura
+
+class ScreenshotService:
+    """
+    Browser Service using DrissionPage.
+    """
+    
+    def __init__(self, headless: bool = True, auto_start: bool = True):
+        self.headless = headless
+        self._manager = None
+        self._executor = ThreadPoolExecutor(max_workers=10)
+        
+        if auto_start:
+            self._ensure_ready()
+    
+    def _ensure_ready(self):
+        """Ensure shared browser is ready."""
+        from .manager import get_shared_browser_manager
+        self._manager = get_shared_browser_manager(headless=self.headless)
+
+    async def fetch_page(self, url: str, timeout: float = 20.0, include_screenshot: bool = True) -> Dict[str, Any]:
+        """
+        Fetch page content (and optionally screenshot).
+        Runs in a thread executor to avoid blocking the async loop.
+        """
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(
+            self._executor,
+            self._fetch_page_sync,
+            url,
+            timeout,
+            include_screenshot
+        )
+
+    def _fetch_page_sync(self, url: str, timeout: float, include_screenshot: bool) -> Dict[str, Any]:
+        """Synchronous fetch logic."""
+        if not url:
+            return {"content": "Error: missing url", "title": "Error", "url": ""}
+        
+        tab = None
+        try:
+            self._ensure_ready()
+            page = self._manager.page
+            if not page:
+                return {"content": "Error: Browser not available", "title": "Error", "url": url}
+            
+            # New Tab
+            tab = page.new_tab(url)
+            
+            # Wait logic
+            is_search_page = any(s in url.lower() for s in ['search', 'bing.com', 'duckduckgo', 'google.com/search', 'searx'])
+            if is_search_page:
+                # Quick check for results
+                result_selectors = ['#results', '#b_results', '#search', '#links', '.result']
+                for selector in result_selectors:
+                    if tab.ele(selector, timeout=1):
+                        break
+            else:
+                # 1. Wait for document to settle (Fast Dynamic Wait)
+                try:
+                    tab.wait.doc_loaded(timeout=5)
+                    # Brief check for loading overlays (fast skip if none)
+                    tab.run_js("""
+                        (async () => {
+                            const isVisible = (el) => !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+                            for (let i = 0; i < 15; i++) {
+                                const indicators = Array.from(document.querySelectorAll('*')).filter(el => {
+                                    try {
+                                        const text = (el.textContent || '').toLowerCase();
+                                        const id = (el.id || '').toLowerCase();
+                                        const cls = (el.getAttribute('class') || '').toLowerCase();
+                                        return (text.includes('loading') || id.includes('loading') || cls.includes('loading')) && isVisible(el);
+                                    } catch(e) { return false; }
+                                });
+                                if (indicators.length === 0) break;
+                                await new Promise(r => setTimeout(r, 100));
+                            }
+                        })()
+                    """, as_expr=True)
+                except: pass
+
+            html = tab.html
+            title = tab.title
+            final_url = tab.url
+            
+            raw_screenshot_b64 = None
+            if include_screenshot:
+                try:
+                    # Scrollbar Hiding Best Effort
+                    from .manager import SharedBrowserManager
+                    SharedBrowserManager.hide_scrollbars(tab)
+                    
+                    # Inject CSS
+                    tab.run_js("""
+                        const style = document.createElement('style');
+                        style.textContent = `
+                            ::-webkit-scrollbar { display: none !important; }
+                            html, body { -ms-overflow-style: none !important; scrollbar-width: none !important; }
+                        `;
+                        document.head.appendChild(style);
+                        document.documentElement.style.overflow = 'hidden';
+                        document.body.style.overflow = 'hidden';
+                    """)
+                    
+                    raw_screenshot_b64 = tab.get_screenshot(as_base64='jpg', full_page=False)
+                except Exception as e:
+                    logger.warning(f"ScreenshotService: Failed to capture screenshot: {e}")
+
+            # Extract content
+            content = trafilatura.extract(
+                html, include_links=True, include_images=True, include_comments=False, 
+                include_tables=True, favor_precision=False, output_format="markdown"
+            ) or ""
+
+            # 2. Extract Images via Parallelized JS (Gallery)
+            # Strategy: For search pages, use Canvas to grab already loaded images (Instant)
+            # For other pages, use fetch (more robust for lazy load)
+            images_b64 = []
+            try:
+                js_code = """
+                    (async () => {
+                        const blocklist = ['logo', 'icon', 'avatar', 'ad', 'pixel', 'tracker', 'button', 'menu', 'nav'];
+                        const candidates = Array.from(document.querySelectorAll('img'));
+                        const validImages = [];
+                        
+                        // Helper: Get base64 from loaded image via Canvas
+                        const getBase64 = (img) => {
+                            try {
+                                const canvas = document.createElement('canvas');
+                                canvas.width = img.naturalWidth;
+                                canvas.height = img.naturalHeight;
+                                const ctx = canvas.getContext('2d');
+                                ctx.drawImage(img, 0, 0);
+                                return canvas.toDataURL('image/jpeg').split(',')[1];
+                            } catch(e) { return null; }
+                        };
+
+                        for (const img of candidates) {
+                            if (validImages.length >= 8) break;
+                            
+                            if (img.naturalWidth < 100 || img.naturalHeight < 80) continue;
+                            
+                            const alt = (img.alt || '').toLowerCase();
+                            const cls = (typeof img.className === 'string' ? img.className : '').toLowerCase();
+                            const src = (img.src || '').toLowerCase();
+                            
+                            if (blocklist.some(b => alt.includes(b) || cls.includes(b) || src.includes(b))) continue;
+                            
+                            // 1. Try Canvas (Instant for loaded images)
+                            if (img.complete && img.naturalHeight > 0) {
+                                const b64 = getBase64(img);
+                                if (b64) {
+                                    validImages.push(b64);
+                                    continue;
+                                }
+                            }
+                            
+                            // 2. Fallback to fetch (only for non-search pages to avoid delay)
+                            // We skip fetch for search pages to ensure speed
+                            if (!window.location.href.includes('google') && !window.location.href.includes('search')) {
+                                try {
+                                    const controller = new AbortController();
+                                    const id = setTimeout(() => controller.abort(), 2000);
+                                    const resp = await fetch(img.src, { signal: controller.signal });
+                                    clearTimeout(id);
+                                    const blob = await resp.blob();
+                                    const b64 = await new Promise(resolve => {
+                                        const reader = new FileReader();
+                                        reader.onloadend = () => resolve(reader.result.split(',')[1]);
+                                        reader.onerror = () => resolve(null);
+                                        reader.readAsDataURL(blob);
+                                    });
+                                    if (b64) validImages.push(b64);
+                                } catch(e) {}
+                            }
+                        }
+                        return validImages;
+                    })()
+                """
+                images_b64 = tab.run_js(js_code, as_expr=True) or []
+                
+                if images_b64:
+                    logger.info(f"ScreenshotService: Extracted {len(images_b64)} images for {url}")
+                
+            except Exception as e:
+                logger.warning(f"ScreenshotService: Image extraction failed: {e}")
+
+            return {
+                "content": content,
+                "html": html,
+                "title": title,
+                "url": final_url,
+                "raw_screenshot_b64": raw_screenshot_b64,
+                "images": images_b64
+            }
+
+        except Exception as e:
+            logger.error(f"ScreenshotService: Failed to fetch {url}: {e}")
+            return {"content": f"Error: fetch failed ({e})", "title": "Error", "url": url}
+        finally:
+            if tab:
+                try: tab.close()
+                except: pass
+
+    async def fetch_pages_batch(self, urls: List[str], timeout: float = 20.0, include_screenshot: bool = True) -> List[Dict[str, Any]]:
+        """Fetch multiple pages concurrently."""
+        if not urls: return []
+        logger.info(f"ScreenshotService: Batch fetching {len(urls)} URLs (screenshots={include_screenshot})")
+        tasks = [self.fetch_page(url, timeout, include_screenshot) for url in urls]
+        return await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def screenshot_url(self, url: str, wait_load: bool = True, timeout: float = 15.0, full_page: bool = False, quality: int = 80) -> Optional[str]:
+        """Screenshot URL (Async wrapper for sync)."""
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(
+            self._executor,
+            self._screenshot_sync,
+            url, wait_load, timeout, full_page, quality
+        )
+
+    def _screenshot_sync(self, url: str, wait_load: bool, timeout: float, full_page: bool, quality: int) -> Optional[str]:
+        """Synchronous screenshot."""
+        if not url: return None
+        tab = None
+        try:
+            self._ensure_ready()
+            page = self._manager.page
+            if not page: return None
+            
+            tab = page.new_tab(url)
+            try:
+                if wait_load:
+                    tab.wait.load_complete(timeout=timeout)
+                else:
+                    tab.wait.doc_loaded(timeout=timeout)
+            except: pass
+            
+            # Wait for main element
+            if tab.ele("#main-container"):
+                pass 
+            
+            # Scrollbar Hiding
+            from .manager import SharedBrowserManager
+            SharedBrowserManager.hide_scrollbars(tab)
+            tab.run_js("""
+                const style = document.createElement('style');
+                style.textContent = `
+                    ::-webkit-scrollbar { display: none !important; }
+                    html, body { -ms-overflow-style: none !important; scrollbar-width: none !important; }
+                `;
+                document.head.appendChild(style);
+                document.documentElement.style.overflow = 'hidden';
+                document.body.style.overflow = 'hidden';
+            """)
+            
+            ele = tab.ele("#main-container")
+            if ele:
+                return ele.get_screenshot(as_base64='jpg', quality=quality)
+            else:
+                return tab.get_screenshot(as_base64='jpg', full_page=full_page, quality=quality)
+                
+        except Exception as e:
+            logger.error(f"ScreenshotService: Screenshot URL failed: {e}")
+            return None
+        finally:
+            if tab:
+                try: tab.close()
+                except: pass
+
+    async def close(self):
+        self._executor.shutdown(wait=False)
+        logger.info("ScreenshotService: Closed.")
+
+    async def close_async(self):
+        await self.close()
+
+# Singleton
+_screenshot_service: Optional[ScreenshotService] = None
+
+def get_screenshot_service(headless: bool = True) -> ScreenshotService:
+    global _screenshot_service
+    if _screenshot_service is None:
+        _screenshot_service = ScreenshotService(headless=headless, auto_start=True)
+    return _screenshot_service
+
+async def close_screenshot_service():
+    global _screenshot_service
+    if _screenshot_service:
+        await _screenshot_service.close()
+        _screenshot_service = None
+
+def prestart_browser(headless: bool = True):
+    get_screenshot_service(headless=headless)