entari-plugin-hyw 4.0.0rc17__py3-none-any.whl → 4.0.0rc19__py3-none-any.whl

hyw_core/browser_control/service.py

@@ -1,1002 +0,0 @@
-"""
-Browser Service (DrissionPage)
-
-Provides page fetching and screenshot capabilities using DrissionPage.
-"""
-
-import asyncio
-import base64
-import threading
-import time
-from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, Dict, Any, List
-from loguru import logger
-import trafilatura
-from PIL import Image
-from io import BytesIO
-
-# Import intelligent completeness checker
-from ..crawling.completeness import CompletenessChecker, trigger_lazy_load
-from ..crawling.models import CrawlConfig
-
-class ScreenshotService:
-    """
-    Browser Service using DrissionPage.
-    """
-    
-    def __init__(self, headless: bool = True, auto_start: bool = True):
-        self.headless = headless
-        self._manager = None
-        self._executor = ThreadPoolExecutor(max_workers=10)
-        
-        if auto_start:
-            self._ensure_ready()
-
-    def _get_tab(self, url: str) -> Any:
-        """Create a new tab and navigate to URL."""
-        self._ensure_ready()
-        return self._manager.new_tab(url)
-
-    def _release_tab(self, tab: Any):
-        """Close tab after use."""
-        if not tab: return
-        try:
-            tab.close()
-        except:
-            pass
-
-    async def search_via_page_input_batch(self, queries: List[str], url: str, selector: str = "#input") -> List[Dict[str, Any]]:
-        """
-        Execute concurrent searches using page inputs.
-        """
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(
-            self._executor,
-            self._search_via_page_input_batch_sync,
-            queries, url, selector
-        )
-
-    def _search_via_page_input_batch_sync(self, queries: List[str], url: str, selector: str) -> List[Dict[str, Any]]:
-        """Sync batch execution - create tabs sequentially, search in parallel."""
-        results = [None] * len(queries)
-        tabs = []
-        
-        # Phase 1: Get/create tabs SEQUENTIALLY (DrissionPage isn't thread-safe for new_tab)
-        target_url = url or "https://www.google.com"
-        logger.info(f"ScreenshotService: Acquiring {len(queries)} tabs for parallel search...")
-        
-        for i in range(len(queries)):
-            tab = None
-            # Try to get from pool first (using shared logic now)
-            try:
-                tab = self._get_tab(target_url)
-            except Exception as e:
-                logger.warning(f"ScreenshotService: Batch search tab creation failed: {e}")
-            
-
-            
-            tabs.append(tab)
-        
-        logger.info(f"ScreenshotService: {len(tabs)} tabs ready, starting parallel searches...")
-        
-        # Phase 2: Execute searches in PARALLEL
-        def run_search(index, tab, query):
-            try:
-                logger.debug(f"Search[{index}]: Starting for '{query}' on {tab.url}")
-                
-                # Wait for page to be ready first
-                try:
-                    tab.wait.doc_loaded(timeout=10)
-                except:
-                    pass
-                
-                # Find input element with wait
-                logger.debug(f"Search[{index}]: Looking for input with selector '{selector}'")
-                ele = tab.ele(selector, timeout=5)
-                if not ele:
-                    logger.debug(f"Search[{index}]: Primary selector failed, trying fallbacks")
-                    for fallback in ["textarea[name='q']", "#APjFqb", "input[name='q']", "input[type='text']"]:
-                        ele = tab.ele(fallback, timeout=2)
-                        if ele:
-                            logger.debug(f"Search[{index}]: Found input with fallback '{fallback}'")
-                            break
-                
-                if not ele:
-                    logger.error(f"Search[{index}]: No input element found on {tab.url}!")
-                    results[index] = {"content": "Error: input not found", "title": "Error", "url": tab.url, "html": tab.html[:5000]}
-                    return
-
-                logger.debug(f"Search[{index}]: Typing query...")
-                ele.input(query)
-                
-                logger.debug(f"Search[{index}]: Pressing Enter...")
-                tab.actions.key_down('enter').key_up('enter')
-                
-                logger.debug(f"Search[{index}]: Waiting for search results...")
-                tab.wait.doc_loaded(timeout=10)
-                # Reduced settle wait for extraction
-                time.sleep(0.1)
-                
-                logger.debug(f"Search[{index}]: Extracting content...")
-                html = tab.html
-                content = trafilatura.extract(
-                    html, include_links=True, include_images=True, include_comments=False,
-                    include_tables=True, favor_precision=False, output_format="markdown"
-                ) or ""
-                
-                logger.info(f"ScreenshotService: Search '{query}' completed -> {tab.url}")
-                
-                results[index] = {
-                    "content": content,
-                    "html": html,
-                    "title": tab.title,
-                    "url": tab.url,
-                    "images": []
-                }
-                
-            except Exception as e:
-                logger.error(f"ScreenshotService: Search error for '{query}': {e}")
-                results[index] = {"content": f"Error: {e}", "title": "Error", "url": "", "html": ""}
-            finally:
-                self._release_tab(tab)
-
-        threads = []
-        for i, (tab, query) in enumerate(zip(tabs, queries)):
-            t = threading.Thread(target=run_search, args=(i, tab, query))
-            t.start()
-            threads.append(t)
-        
-        for t in threads:
-            t.join()
-            
-        return results
-    
-    def _ensure_ready(self):
-        """Ensure shared browser is ready."""
-        from .manager import get_shared_browser_manager
-        self._manager = get_shared_browser_manager(headless=self.headless)
-
-    async def fetch_page(self, url: str, timeout: float = 10.0, include_screenshot: bool = True) -> Dict[str, Any]:
-        """
-        Fetch page content (and optionally screenshot).
-        Runs in a thread executor to avoid blocking the async loop.
-        """
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(
-            self._executor,
-            self._fetch_page_sync,
-            url,
-            timeout,
-            include_screenshot
-        )
-
-    async def search_via_address_bar(self, query: str, timeout: float = 20.0) -> Dict[str, Any]:
-        """
-        Search using browser's address bar (uses browser's default search engine).
-        Simulates: Ctrl+L (focus address bar) -> type query -> Enter
-        """
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(
-            self._executor,
-            self._search_via_address_bar_sync,
-            query,
-            timeout
-        )
-    
-    def _search_via_address_bar_sync(self, query: str, timeout: float) -> Dict[str, Any]:
-        """Synchronous address bar search logic."""
-        if not query:
-            return {"content": "Error: missing query", "title": "Error", "url": "", "html": ""}
-        
-        tab = None
-        try:
-            self._ensure_ready()
-            page = self._manager.page
-            if not page:
-                return {"content": "Error: Browser not available", "title": "Error", "url": "", "html": ""}
-            
-            # Open new blank tab
-            tab = page.new_tab()
-            
-            # Focus address bar with Ctrl+L (or Cmd+L on Mac)
-            import platform
-            if platform.system() == "Darwin":
-                tab.actions.key_down('cmd').key_down('l').key_up('l').key_up('cmd')
-            else:
-                tab.actions.key_down('ctrl').key_down('l').key_up('l').key_up('ctrl')
-            
-            # Small delay for address bar to focus
-            import time as _time
-            _time.sleep(0.05)
-            
-            # Type the query
-            tab.actions.type(query)
-            
-            # Press Enter to search
-            tab.actions.key_down('enter').key_up('enter')
-            
-            # Wait for page to load
-            try:
-                tab.wait.doc_loaded(timeout=timeout)
-                # Reduced wait for initial results
-                _time.sleep(0.2)
-            except:
-                pass
-            
-            html = tab.html
-            title = tab.title
-            final_url = tab.url
-            
-            # Extract content
-            content = trafilatura.extract(
-                html, include_links=True, include_images=True, include_comments=False,
-                include_tables=True, favor_precision=False, output_format="markdown"
-            ) or ""
-            
-            logger.info(f"ScreenshotService: Address bar search completed -> {final_url}")
-            
-            return {
-                "content": content,
-                "html": html,
-                "title": title,
-                "url": final_url,
-                "images": []
-            }
-            
-        except Exception as e:
-            logger.error(f"ScreenshotService: Address bar search failed: {e}")
-            return {"content": f"Error: search failed ({e})", "title": "Error", "url": "", "html": ""}
-        finally:
-            if tab:
-                try: tab.close()
-                except: pass
-
-    def _scroll_to_bottom(self, tab, step: int = 800, delay: float = 2.0, timeout: float = 10.0):
-        """
-        Scroll down gradually to trigger lazy loading.
-        
-        Args:
-            delay: Max wait time per scroll step (seconds) if images aren't loading.
-        """
-        import time
-        start = time.time()
-        current_pos = 0
-        try:
-            while time.time() - start < timeout:
-                # Scroll down
-                current_pos += step
-                tab.run_js(f"window.scrollTo(0, {current_pos});")
-                
-                # Active Wait: Check if images in viewport are loaded
-                # Poll every 100ms, up to 'delay' seconds
-                wait_start = time.time()
-                while time.time() - wait_start < delay:
-                    all_loaded = tab.run_js("""
-                        return (async () => {
-                            const imgs = Array.from(document.querySelectorAll('img'));
-                            const viewportHeight = window.innerHeight;
-                            
-                            // 1. Identify images currently in viewport
-                            const visibleImgs = imgs.filter(img => {
-                                const rect = img.getBoundingClientRect();
-                                return (rect.top < viewportHeight && rect.bottom > 0) && (rect.width > 0 && rect.height > 0);
-                            });
-                            
-                            if (visibleImgs.length === 0) return true;
-
-                            // 2. Check loading status using decode() AND heuristic for placeholders
-                            // Some sites load a tiny blurred placeholder first. 
-                            const checks = visibleImgs.map(img => {
-                                // Enhanced placeholder detection (matching completeness.py)
-                                const dataSrc = img.getAttribute('data-src') || img.getAttribute('data-original') || 
-                                                img.getAttribute('data-lazy-src') || img.getAttribute('data-lazy') || '';
-                                const className = (typeof img.className === 'string' ? img.className : '').toLowerCase();
-                                const loadingAttr = img.getAttribute('loading') || '';
-                                const src = img.src || '';
-                                
-                                const isPlaceholder = (
-                                    // data-src not yet loaded
-                                    (dataSrc && img.src !== dataSrc) ||
-                                    // Natural size much smaller than display (blurred placeholder)
-                                    (img.naturalWidth < 50 && img.clientWidth > 100) ||
-                                    (img.naturalWidth < 100 && img.clientWidth > 200 && img.naturalWidth * 4 < img.clientWidth) ||
-                                    // Lazy-loading class indicators
-                                    className.includes('lazy') ||
-                                    className.includes('lazyload') ||
-                                    className.includes('lozad') ||
-                                    // CSS blur filter applied
-                                    (window.getComputedStyle(img).filter || '').includes('blur') ||
-                                    // loading="lazy" + not complete
-                                    (loadingAttr === 'lazy' && !img.complete)
-                                );
-                                
-                                if (isPlaceholder) {
-                                    // If it looks like a placeholder, we return false (not loaded)
-                                    // unless it stays like this for too long (handled by outer timeout)
-                                    return Promise.resolve(false);
-                                }
-
-                                if (img.complete && img.naturalHeight > 0) return Promise.resolve(true); 
-                                
-                                return img.decode().then(() => true).catch(() => false); 
-                            });
-                            
-                            // Race against a small timeout to avoid hanging on one broken image
-                            const allDecoded = Promise.all(checks);
-                            const timeout = new Promise(resolve => setTimeout(() => resolve(false), 500));
-                            
-                            // If any check returned false (meaning placeholder or not decoded), result is false
-                            return Promise.race([allDecoded, timeout]).then(results => {
-                                if (!Array.isArray(results)) return results === true;
-                                return results.every(res => res === true);
-                            }); 
-                        })();
-                    """)
-                    if all_loaded:
-                        break
-                    time.sleep(0.1)
-
-                # Check if reached bottom
-                height = tab.run_js("return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);")
-                if current_pos >= height:
-                    break
-            
-            # Ensure final layout settle
-            time.sleep(0.2)
-            
-        except Exception as e:
-            logger.warning(f"ScreenshotService: Scroll failed: {e}")
-
-    def _fetch_page_sync(self, url: str, timeout: float, include_screenshot: bool) -> Dict[str, Any]:
-        """Synchronous fetch logic."""
-        if not url:
-            return {"content": "Error: missing url", "title": "Error", "url": ""}
-        
-        tab = None
-        try:
-            self._ensure_ready()
-            page = self._manager.page
-            if not page:
-                return {"content": "Error: Browser not available", "title": "Error", "url": url}
-            
-            # Get from pool
-            tab = self._get_tab(url)
-            
-            # Wait logic - optimized for search pages
-            is_search_page = any(s in url.lower() for s in ['search', 'bing.com', 'duckduckgo', 'google.com/search', 'searx'])
-            if is_search_page:
-                # Optimized waiting: Rapidly poll for ACTUAL results > 0
-                start_time = time.time()
-                
-                # Special fast-path for DDG Lite (HTML only, no JS rendering needed)
-                if 'lite.duckduckgo' in url:
-                    # just wait for body, it's static HTML
-                     try:
-                         tab.wait.doc_loaded(timeout=timeout)
-                     except: pass
-                     # Sleep tiny bit to ensure render
-                     time.sleep(0.5)
-                else:
-                    while time.time() - start_time < timeout:
-                        found_results = False
-                        try:
-                            if 'google' in url.lower():
-                                # Check if we have any result items (.g, .MjjYud) or the main container (#search)
-                                # Using checks with minimal timeout to allow fast looping
-                                if tab.ele('.g', timeout=0.1) or tab.ele('.MjjYud', timeout=0.1) or tab.ele('#search', timeout=0.1):
-                                    found_results = True
-                            elif 'bing' in url.lower():
-                                if tab.ele('.b_algo', timeout=0.1) or tab.ele('#b_results', timeout=0.1):
-                                    found_results = True
-                            elif 'duckduckgo' in url.lower():
-                                if tab.ele('.result', timeout=0.1) or tab.ele('#react-layout', timeout=0.1):
-                                    found_results = True
-                            else:
-                                # Generic fallback: wait for body to be populated
-                                if tab.ele('body', timeout=0.1):
-                                    found_results = True
-                        except:
-                            pass
-                        
-                        if found_results:
-                            break
-                        time.sleep(0.05)  # Faster polling (50ms) as requested
-            else:
-                # 1. Wait for document to settle (Fast Dynamic Wait)
-                try:
-                    tab.wait.doc_loaded(timeout=timeout)
-                except: pass
-
-            html = tab.html
-            title = tab.title
-            final_url = tab.url
-            
-            raw_screenshot_b64 = None
-            if include_screenshot:
-                try:
-                    # Scrollbar Hiding Best Effort
-                    from .manager import SharedBrowserManager
-                    SharedBrowserManager.hide_scrollbars(tab)
-                    
-                    # Inject CSS
-                    tab.run_js("""
-                        const style = document.createElement('style');
-                        style.textContent = `
-                            ::-webkit-scrollbar { display: none !important; }
-                            html, body { -ms-overflow-style: none !important; scrollbar-width: none !important; }
-                        `;
-                        document.head.appendChild(style);
-                        document.documentElement.style.overflow = 'hidden';
-                        document.body.style.overflow = 'hidden';
-                    """)
-                    
-                    raw_screenshot_b64 = tab.get_screenshot(as_base64='jpg', full_page=False)
-                except Exception as e:
-                    logger.warning(f"ScreenshotService: Failed to capture screenshot: {e}")
-
-            # Extract content
-            content = trafilatura.extract(
-                html, include_links=True, include_images=True, include_comments=False, 
-                include_tables=True, favor_precision=False, output_format="markdown"
-            ) or ""
-
-            # 2. Extract Images via Parallelized JS (Gallery)
-            # Strategy: For search pages, use Canvas to grab already loaded images (Instant)
-            # For other pages, use fetch (more robust for lazy load)
-            images_b64 = []
-            try:
-                js_code = """
-                    (async () => {
-                        const blocklist = ['logo', 'icon', 'avatar', 'ad', 'pixel', 'tracker', 'button', 'menu', 'nav'];
-                        const candidates = Array.from(document.querySelectorAll('img'));
-                        const validImages = [];
-                        
-                        // Helper: Get base64 from loaded image via Canvas
-                        const getBase64 = (img) => {
-                            try {
-                                const canvas = document.createElement('canvas');
-                                canvas.width = img.naturalWidth;
-                                canvas.height = img.naturalHeight;
-                                const ctx = canvas.getContext('2d');
-                                ctx.drawImage(img, 0, 0);
-                                return canvas.toDataURL('image/jpeg').split(',')[1];
-                            } catch(e) { return null; }
-                        };
-
-                        for (const img of candidates) {
-                            if (validImages.length >= 8) break;
-                            
-                            if (img.naturalWidth < 100 || img.naturalHeight < 80) continue;
-                            
-                            const alt = (img.alt || '').toLowerCase();
-                            const cls = (typeof img.className === 'string' ? img.className : '').toLowerCase();
-                            const src = (img.src || '').toLowerCase();
-                            
-                            if (blocklist.some(b => alt.includes(b) || cls.includes(b) || src.includes(b))) continue;
-                            
-                            // 1. Try Canvas (Instant for loaded images)
-                            if (img.complete && img.naturalHeight > 0) {
-                                const b64 = getBase64(img);
-                                if (b64) {
-                                    validImages.push(b64);
-                                    continue;
-                                }
-                            }
-                            
-                            // 2. Fallback to fetch (only for non-search pages to avoid delay)
-                            // We skip fetch for search pages to ensure speed
-                            if (!window.location.href.includes('google') && !window.location.href.includes('search')) {
-                                try {
-                                    const controller = new AbortController();
-                                    const id = setTimeout(() => controller.abort(), 2000);
-                                    const resp = await fetch(img.src, { signal: controller.signal });
-                                    clearTimeout(id);
-                                    const blob = await resp.blob();
-                                    const b64 = await new Promise(resolve => {
-                                        const reader = new FileReader();
-                                        reader.onloadend = () => resolve(reader.result.split(',')[1]);
-                                        reader.onerror = () => resolve(null);
-                                        reader.readAsDataURL(blob);
-                                    });
-                                    if (b64) validImages.push(b64);
-                                } catch(e) {}
-                            }
-                        }
-                        return validImages;
-                    })()
-                """
-                images_b64 = tab.run_js(js_code, as_expr=True) or []
-                
-                if images_b64:
-                    logger.info(f"ScreenshotService: Extracted {len(images_b64)} images for {url}")
-                
-            except Exception as e:
-                logger.warning(f"ScreenshotService: Image extraction failed: {e}")
-
-            return {
-                "content": content,
-                "html": html,
-                "title": title,
-                "url": final_url,
-                "raw_screenshot_b64": raw_screenshot_b64,
-                "images": images_b64
-            }
-
-        except Exception as e:
-            logger.error(f"ScreenshotService: Failed to fetch {url}: {e}")
-            return {"content": f"Error: fetch failed ({e})", "title": "Error", "url": url}
-        finally:
-            if tab:
-               self._release_tab(tab)
-
-    async def fetch_pages_batch(self, urls: List[str], timeout: float = 20.0, include_screenshot: bool = True) -> List[Dict[str, Any]]:
-        """Fetch multiple pages concurrently."""
-        if not urls: return []
-        logger.info(f"ScreenshotService: Batch fetching {len(urls)} URLs (screenshots={include_screenshot})")
-        tasks = [self.fetch_page(url, timeout, include_screenshot) for url in urls]
-        return await asyncio.gather(*tasks, return_exceptions=True)
-
-    async def screenshot_urls_batch(self, urls: List[str], timeout: float = 15.0, full_page: bool = True) -> List[Optional[str]]:
-        """Take screenshots of multiple URLs concurrently."""
-        if not urls: return []
-        logger.info(f"ScreenshotService: Batch screenshot {len(urls)} URLs")
-        tasks = [self.screenshot_url(url, timeout=timeout, full_page=full_page) for url in urls]
-        return await asyncio.gather(*tasks, return_exceptions=True)
-
-    async def screenshot_url(self, url: str, wait_load: bool = True, timeout: float = 15.0, full_page: bool = False, quality: int = 90, scale: int = 1) -> Optional[str]:
-        """Screenshot URL (Async wrapper for sync). Returns base64 string only."""
-        loop = asyncio.get_running_loop()
-        result = await loop.run_in_executor(
-            self._executor,
-            self._screenshot_sync,
-            url, wait_load, timeout, full_page, quality, scale, False  # extract_content=False
-        )
-        # Backward compatible: return just the screenshot for old callers
-        if isinstance(result, dict):
-            return result.get("screenshot_b64")
-        return result
-
-    async def screenshot_with_content(self, url: str, timeout: float = 15.0, max_content_length: int = 8000) -> Dict[str, Any]:
-        """
-        Screenshot URL and extract page content.
-        
-        Returns:
-            Dict with:
-                - screenshot_b64: base64 encoded screenshot
-                - content: trafilatura extracted text (truncated to max_content_length)
-                - title: page title
-                - url: final URL
-        """
-        loop = asyncio.get_running_loop()
-        result = await loop.run_in_executor(
-            self._executor,
-            self._screenshot_sync,
-            url, True, timeout, False, 80, 2, True  # quality=80 for balance, scale=2, extract_content=True
-        )
-        
-        if not isinstance(result, dict):
-            return {"screenshot_b64": result, "content": "", "title": "", "url": url}
-        
-        # Truncate content if needed
-        content = result.get("content", "") or ""
-        if len(content) > max_content_length:
-            content = content[:max_content_length] + "\n\n[内容已截断...]"
-        result["content"] = content
-        
-        return result
-
-
-    def _screenshot_sync(self, url: str, wait_load: bool, timeout: float, full_page: bool, quality: int, scale: int = 1, extract_content: bool = False) -> Any:
-        """Synchronous screenshot. If extract_content=True, returns Dict else str."""
-        if not url: 
-            return {"screenshot_b64": None, "content": "", "title": "", "url": url} if extract_content else None
-        tab = None
-        capture_width = 3000  # Increased for more comfortable page size while maintaining high resolution
-        
-        try:
-            self._ensure_ready()
-            page = self._manager.page
-            if not page: return None
-            
-            # Create blank tab first
-            tab = page.new_tab()
-            
-            # Set viewport BEFORE navigation so page renders at target width from the start
-            # This eliminates the need for post-load resize and reflow
-            try:
-                tab.run_cdp('Emulation.setDeviceMetricsOverride', 
-                            width=capture_width, height=900, deviceScaleFactor=scale, mobile=False)
-            except:
-                pass
-            
-            # Now navigate to the URL - page will render at target width
-            tab.get(url)
-            
-            # Network monitoring removed - not needed for simplified wait logic
-
-            # Initialize crawl config for completeness checking (defined outside try for scope)
-            crawl_config = CrawlConfig(
-                scan_full_page=True,
-                scroll_step=800,
-                scroll_delay=0.5,
-                scroll_timeout=20.0,  # Increased for lazy-loading pages
-                image_load_timeout=3.0,  # Image loading timeout: 3 seconds
-                image_stability_checks=3,
-            )
-
-            # === Start scrolling immediately to trigger lazy loading ===
-            # Scroll first, then wait for DOM - this allows lazy loading to start in parallel
-            logger.info(f"ScreenshotService: Starting lazy load scroll for {url} (before DOM wait)")
-            trigger_lazy_load(tab, crawl_config)
-
-            # Now wait for DOM to be ready (after scroll has triggered lazy loading)
-            try:
-                # Wait for full page load (including JS execution)
-                try:
-                    tab.wait.doc_loaded(timeout=timeout)
-                except:
-                    pass
-                
-                # Wait for actual content to appear (for CDN verification pages)
-                # Simplified wait logic for screenshot: just check basic readiness
-                
-                for i in range(20):  # Max 20 iterations (~1s) - much faster
-                    try:
-                        state = tab.run_js('''
-                            return {
-                                ready: document.readyState === 'complete',
-                                title: document.title,
-                                text: document.body.innerText.substring(0, 500) || ""
-                            };
-                        ''') or {'ready': False, 'title': "", 'text': ""}
-                        
-                        is_ready = state.get('ready', False)
-                        title = state.get('title', "").lower()
-                        text_lower = state.get('text', "").lower()
-                        text_len = len(text_lower)
-                        
-                        # Check for verification pages
-                        is_verification = "checking your browser" in text_lower or \
-                                        "just a moment" in text_lower or \
-                                        "please wait" in text_lower or \
-                                        "security check" in title or \
-                                        "just a moment" in title or \
-                                        "loading..." in title
-                        
-                        # Basic content check
-                        has_content = text_len > 100
-                        
-                        # Pass if ready, not verification, and has content
-                        if is_ready and not is_verification and has_content:
-                            logger.debug(f"ScreenshotService: Page ready after {i * 0.05:.2f}s")
-                            break
-                        
-                        time.sleep(0.05)
-                    except Exception:
-                        time.sleep(0.05)
-                        continue
-                
-                # DEBUG: Save HTML to inspect what happened (in data dir)
-                try:
-                    import os
-                    log_path = os.path.join(os.getcwd(), "data", "browser.log.html")
-                    with open(log_path, "w", encoding="utf-8") as f:
-                        f.write(f"<!-- URL: {url} -->\n")
-                        f.write(tab.html)
-                except: pass
-
-            except Exception as e:
-                logger.warning(f"ScreenshotService: Page readiness check failed: {e}")
-            
-            # Scrollbar Hiding first (before any height calculation)
-            from .manager import SharedBrowserManager
-            SharedBrowserManager.hide_scrollbars(tab)
-            
-            # Scroll back to top
-            tab.run_js("window.scrollTo(0, 0);")
-            
-            # Image loading monitoring with time tracking - DISABLED
-            # No longer waiting for images to load
-            # Initialize image tracking JavaScript
-            # image_tracking_js = """
-            # (() => {
-            #     if (!window._imageLoadTracker) {
-            #         window._imageLoadTracker = {
-            #             startTime: Date.now(),
-            #             images: new Map()
-            #         };
-            #         
-            #         const imgs = Array.from(document.querySelectorAll('img'));
-            #         const minSize = 50;
-            #         
-            #         imgs.forEach((img, idx) => {
-            #             if (img.clientWidth < minSize && img.clientHeight < minSize) return;
-            #             
-            #             const src = img.src || img.getAttribute('data-src') || '';
-            #             const key = `${idx}_${src.substring(0, 100)}`;
-            #             
-            #             if (img.complete && img.naturalWidth > 0 && img.naturalHeight > 0) {
-            #                 // Already loaded
-            #                 window._imageLoadTracker.images.set(key, {
-            #                     src: src.substring(0, 150),
-            #                     status: 'loaded',
-            #                     loadTime: 0,  // Already loaded before tracking
-            #                     naturalSize: [img.naturalWidth, img.naturalHeight],
-            #                     displaySize: [img.clientWidth, img.clientHeight]
-            #                 });
-            #             } else {
-            #                 // Track loading
-            #                 window._imageLoadTracker.images.set(key, {
-            #                     src: src.substring(0, 150),
-            #                     status: 'pending',
-            #                     startTime: Date.now(),
-            #                     naturalSize: [img.naturalWidth, img.naturalHeight],
-            #                     displaySize: [img.clientWidth, img.clientHeight]
-            #                 });
-            #                 
-            #                 // Add load event listener
-            #                 img.addEventListener('load', () => {
-            #                     const entry = window._imageLoadTracker.images.get(key);
-            #                     if (entry && entry.status === 'pending') {
-            #                         entry.status = 'loaded';
-            #                         entry.loadTime = Date.now() - entry.startTime;
-            #                     }
-            #                 });
-            #                 
-            #                 img.addEventListener('error', () => {
-            #                     const entry = window._imageLoadTracker.images.get(key);
-            #                     if (entry && entry.status === 'pending') {
-            #                         entry.status = 'failed';
-            #                         entry.loadTime = Date.now() - entry.startTime;
-            #                     }
-            #                 });
-            #             }
-            #         });
-            #     }
-            #     
-            #     // Return current status
-            #     const results = [];
-            #     window._imageLoadTracker.images.forEach((value, key) => {
-            #         const entry = {
-            #             src: value.src,
-            #             status: value.status,
-            #             loadTime: value.status === 'loaded' ? (value.loadTime || 0) : (Date.now() - value.startTime),
-            #             naturalSize: value.naturalSize,
-            #             displaySize: value.displaySize
-            #         };
-            #         results.push(entry);
-            #     });
-            #     
-            #     return {
-            #         total: results.length,
-            #         loaded: results.filter(r => r.status === 'loaded').length,
-            #         pending: results.filter(r => r.status === 'pending').length,
-            #         failed: results.filter(r => r.status === 'failed').length,
-            #         details: results
-            #     };
-            # })()
-            # """
-            # 
-            # # Initialize tracking
-            # tab.run_js(image_tracking_js)
-            # 
-            # # Monitor image loading with dynamic stop logic
-            # check_interval = 0.2  # Check every 200ms
-            # image_timeout = 3.0  # Image loading timeout: 3 seconds
-            # monitoring_start = time.time()
-            # loaded_times = []  # Track load times of completed images
-            # 
-            # logger.info(f"ScreenshotService: Starting image load monitoring (timeout={image_timeout}s)...")
-            # 
-            # while True:
-            #     elapsed = time.time() - monitoring_start
-            #     
-            #     # Check timeout first
-            #     if elapsed >= image_timeout:
-            #         logger.info(f"ScreenshotService: Image loading timeout ({image_timeout}s) reached")
-            #         break
-            #     
-            #     # Get current image status
-            #     status = tab.run_js(image_tracking_js, as_expr=True) or {
-            #         'total': 0, 'loaded': 0, 'pending': 0, 'failed': 0, 'details': []
-            #     }
-            #     
-            #     # Log each image's status and load time
-            #     for img_detail in status.get('details', []):
-            #         src_short = img_detail.get('src', '')[:80]
-            #         status_str = img_detail.get('status', 'unknown')
-            #         load_time = img_detail.get('loadTime', 0)
-            #         logger.info(
-            #             f"ScreenshotService: Image [{status_str}] "
-            #             f"loadTime={load_time:.0f}ms "
-            #             f"src={src_short}"
-            #         )
-            #     
-            #     # Collect load times of completed images
-            #     loaded_times = [
-            #         img.get('loadTime', 0) 
-            #         for img in status.get('details', []) 
-            #         if img.get('status') == 'loaded' and img.get('loadTime', 0) > 0
-            #     ]
-            #     
-            #     pending_count = status.get('pending', 0)
-            #     loaded_count = status.get('loaded', 0)
-            #     
-            #     # Check stop conditions
-            #     if pending_count == 0:
-            #         logger.info(f"ScreenshotService: All images loaded. Total: {status.get('total', 0)}, Loaded: {loaded_count}")
-            #         break
-            #     
-            #     # Check dynamic stop condition (if we have loaded images to calculate average)
-            #     if loaded_times:
-            #         avg_load_time = sum(loaded_times) / len(loaded_times)
-            #         max_wait_time = avg_load_time * 2
-            #         
-            #         # Check if any pending image has exceeded max wait time
-            #         pending_images = [
-            #             img for img in status.get('details', [])
-            #             if img.get('status') == 'pending'
-            #         ]
-            #         
-            #         should_stop = False
-            #         for pending_img in pending_images:
-            #             wait_time = pending_img.get('loadTime', 0)
-            #             if wait_time >= max_wait_time:
-            #                 should_stop = True
-            #                 logger.info(
-            #                     f"ScreenshotService: Stopping - pending image waited {wait_time:.0f}ms, "
-            #                     f"exceeds 2x avg load time ({max_wait_time:.0f}ms, avg={avg_load_time:.0f}ms)"
-            #                 )
-            #                 break
-            #         
-            #         if should_stop:
-            #             break
-            #     
-            #     # Wait before next check
-            #     time.sleep(check_interval)
-            
-            # Now calculate final height ONCE after all content loaded
-            # CompletenessChecker already verified height stability
-            try:
-                final_height = tab.run_js('''
-                    return Math.max(
-                        document.body.scrollHeight || 0,
-                        document.documentElement.scrollHeight || 0,
-                        document.body.offsetHeight || 0,
-                        document.documentElement.offsetHeight || 0
-                    );
-                ''')
-                h = min(int(final_height) + 50, 15000)
-                tab.run_cdp('Emulation.setDeviceMetricsOverride', 
-                            width=capture_width, height=h, deviceScaleFactor=1, mobile=False)
-            except:
-                pass
-            
-            # Final scroll to top
-            tab.run_js("window.scrollTo(0, 0);")
-            
-            # Capture screenshot
-            screenshot_b64 = tab.get_screenshot(as_base64='jpg', full_page=False)
-            
-            # Use Pillow for intelligent compression
-            if screenshot_b64 and quality < 95: # Only compress if quality is not near maximum
-                try:
-                    img_bytes = base64.b64decode(screenshot_b64)
-                    img = Image.open(BytesIO(img_bytes))
-                    
-                    # Convert to RGB if not already (some images might be RGBA, which JPEG doesn't support)
-                    if img.mode in ("RGBA", "P"):
-                        img = img.convert("RGB")
-                    
-                    output_buffer = BytesIO()
-                    img.save(output_buffer, format="WebP", quality=quality, optimize=True) # Output as WebP format
-                    screenshot_b64 = base64.b64encode(output_buffer.getvalue()).decode()
-                    logger.debug(f"ScreenshotService: Applied Pillow compression with quality={quality}")
-                except Exception as e:
-                    logger.warning(f"ScreenshotService: Pillow compression failed: {e}")
-            
-            # Extract content if requested
-            if extract_content:
-                try:
-                    html = tab.html
-                    title = tab.title
-                    final_url = tab.url
-                    
-                    # Minimal trafilatura settings to reduce token consumption
-                    content = trafilatura.extract(
-                        html,
-                        include_links=False,     # No links to reduce tokens
-                        include_images=False,    # No image descriptions
-                        include_comments=False,  # No comments
-                        include_tables=False,    # No tables (can be verbose)
-                        favor_precision=True,    # Favor precision over recall
-                        output_format="txt"      # Plain text (no markdown formatting)
-                    ) or ""
-                    
-                    return {
-                        "screenshot_b64": screenshot_b64,
-                        "content": content,
-                        "title": title,
-                        "url": final_url
-                    }
-                except Exception as e:
-                    logger.warning(f"ScreenshotService: Content extraction failed: {e}")
-                    return {"screenshot_b64": screenshot_b64, "content": "", "title": "", "url": url}
-            
-            return screenshot_b64
-                
-        except Exception as e:
-            logger.error(f"ScreenshotService: Screenshot URL failed: {e}")
-            return {"screenshot_b64": None, "content": "", "title": "", "url": url} if extract_content else None
-        finally:
-            if tab:
-                try: tab.close()
-                except: pass
-
-
-    async def execute_script(self, script: str) -> Dict[str, Any]:
-        """
-        Execute JavaScript in the current active page context.
-        This reuses the shared browser instance.
-        """
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(
-            self._executor,
-            self._execute_script_sync,
-            script
-        )
-
-    def _execute_script_sync(self, script: str) -> Dict[str, Any]:
-        """Synchronous JS execution."""
-        try:
-            self._ensure_ready()
-            page = self._manager.page
-            if not page:
-                return {"success": False, "error": "Browser not available"}
-
-            # Get current active tab or first tab
-            # Fix: ChromiumPage object has no attribute 'tabs'
-            # We use the page object itself as it represents the active tab controller
-            tab = page
-            if not tab:
-                return {"success": False, "error": "No active tab"}
-
-            logger.info(f"ScreenshotService: Executing JS on {tab.url}")
-
-            # Execute JS
-            result = tab.run_js(script)
-
-            return {
-                "success": True,
-                "result": result,
-                "url": tab.url,
-                "title": tab.title
-            }
-        except Exception as e:
-            logger.error(f"ScreenshotService: JS execution failed: {e}")
-            return {"success": False, "error": str(e)}
-
-    async def close(self):
-        self._executor.shutdown(wait=False)
-        logger.info("ScreenshotService: Closed.")
-
-    async def close_async(self):
-        await self.close()
-
-# Singleton
-_screenshot_service: Optional[ScreenshotService] = None
-
-def get_screenshot_service(headless: bool = True) -> ScreenshotService:
-    global _screenshot_service
-    if _screenshot_service is None:
-        _screenshot_service = ScreenshotService(headless=headless, auto_start=True)
-    return _screenshot_service
-
-async def close_screenshot_service():
-    global _screenshot_service
-    if _screenshot_service:
-        await _screenshot_service.close()
-        _screenshot_service = None
-
-def prestart_browser(headless: bool = True):
-    get_screenshot_service(headless=headless)