entari-plugin-hyw 4.0.0rc9__py3-none-any.whl → 4.0.0rc10__py3-none-any.whl

hyw_core/browser_control/engines/__init__.py

@@ -5,13 +5,11 @@ Provides search engine adapters for different search providers.
 """
 
 from .base import SearchEngine
-from .google import GoogleEngine
 from .duckduckgo import DuckDuckGoEngine
 from .default import DefaultEngine
 
 __all__ = [
     "SearchEngine",
-    "GoogleEngine", 
     "DuckDuckGoEngine",
     "DefaultEngine",
 ]

hyw_core/browser_control/manager.py

@@ -124,15 +124,28 @@ class SharedBrowserManager:
     @staticmethod
     def hide_scrollbars(page: ChromiumPage):
         """
-        Robustly hide scrollbars using CDP commands.
-        This eliminates the reserved space/gutter that standard CSS might miss.
+        Robustly hide scrollbars using CDP commands AND CSS injection.
+        This provides double protection against scrollbar gutters.
         """
         try:
-            # Emulation.setScrollbarsHidden is a CDP command
+            # 1. CDP Command
             page.run_cdp('Emulation.setScrollbarsHidden', hidden=True)
-            logger.debug("SharedBrowserManager: CDP scrollbars hidden.")
+            
+            # 2. CSS Injection (Standard + Webkit)
+            css = """
+                ::-webkit-scrollbar { display: none !important; width: 0 !important; height: 0 !important; }
+                * { -ms-overflow-style: none !important; scrollbar-width: none !important; }
+            """
+            # Inject into current page
+            page.run_js(f"""
+                const style = document.createElement('style');
+                style.textContent = `{css}`;
+                document.head.appendChild(style);
+            """)
+            
+            logger.debug("SharedBrowserManager: Scrollbars hidden via CDP + CSS.")
         except Exception as e:
-            logger.warning(f"SharedBrowserManager: Failed to hide scrollbars via CDP: {e}")
+            logger.warning(f"SharedBrowserManager: Failed to hide scrollbars: {e}")
 
 
 # Module-level singleton accessor

hyw_core/browser_control/renderer.py

@@ -55,16 +55,43 @@ class ContentRenderer:
         loop = asyncio.get_running_loop()
         return await loop.run_in_executor(self._executor, self._prepare_tab_sync)
 
-    def _wait_for_render_finished(self, tab, timeout: float = 3.0):
+    def _wait_for_render_finished(self, tab, timeout: float = 12.0, context: str = ""):
         """Wait for window.RENDER_FINISHED to be true in the tab."""
         import time as pytime
         start = pytime.time()
+        
+        # Check initial state
+        initial_state = tab.run_js("return window.RENDER_FINISHED")
+        logger.debug(f"ContentRenderer[{context}]: Starting wait, initial RENDER_FINISHED={initial_state}")
+        
+        # If already true, it's stale from previous render - need to wait for JS to reset it
+        if initial_state:
+            logger.debug(f"ContentRenderer[{context}]: RENDER_FINISHED was true, waiting for reset...")
+            # Wait for JS to reset it to false (updateRenderData sets it to false)
+            reset_start = pytime.time()
+            while pytime.time() - reset_start < 1.0:  # 1s max to wait for reset
+                is_reset = tab.run_js("return window.RENDER_FINISHED")
+                if not is_reset:
+                    logger.debug(f"ContentRenderer[{context}]: RENDER_FINISHED reset to false")
+                    break
+                pytime.sleep(0.05)
+            else:
+                logger.warning(f"ContentRenderer[{context}]: RENDER_FINISHED not reset, force resetting via JS")
+                tab.run_js("window.RENDER_FINISHED = false")
+        
+        # Now wait for it to become true
+        poll_count = 0
         while pytime.time() - start < timeout:
             is_finished = tab.run_js("return window.RENDER_FINISHED")
+            poll_count += 1
             if is_finished:
+                elapsed = pytime.time() - start
+                logger.debug(f"ContentRenderer[{context}]: RENDER_FINISHED=true after {elapsed:.2f}s ({poll_count} polls)")
                 return True
-            pytime.sleep(0.05) # Fast polling
-        logger.warning(f"ContentRenderer: Wait for RENDER_FINISHED timed out after {timeout}s")
+            pytime.sleep(0.1)  # Poll every 100ms
+        
+        elapsed = pytime.time() - start
+        logger.warning(f"ContentRenderer[{context}]: Wait for RENDER_FINISHED timed out after {elapsed:.2f}s ({poll_count} polls)")
         return False
 
     def _prepare_tab_sync(self) -> str:
@@ -89,8 +116,9 @@ class ContentRenderer:
                 "theme_color": "#ef4444",
             }
             
+            logger.debug(f"ContentRenderer: Calling warmup updateRenderData for tab {tab_id}")
             tab.run_js(f"window.updateRenderData({json.dumps(warmup_data)})")
-            self._wait_for_render_finished(tab, timeout=5.0)
+            self._wait_for_render_finished(tab, timeout=12.0, context=f"warmup:{tab_id}")
             
             # Wait for main-container after warmup (Vue needs to render it)
             tab.ele('#main-container', timeout=3)
@@ -203,7 +231,7 @@ class ContentRenderer:
             
             # 1. Update Data & Wait for Finished flag
             tab.run_js(f"window.updateRenderData({json.dumps(render_data)})")
-            self._wait_for_render_finished(tab)
+            self._wait_for_render_finished(tab, context=f"batch:{tab_id}")
 
             # 2. Dynamic Resize
             # Get actual content height to prevent clipping
@@ -330,10 +358,12 @@ class ContentRenderer:
                 "theme_color": theme_color,
             }
             
+            actual_tab_id = getattr(tab, 'tab_id', 'unknown')
+            logger.info(f"ContentRenderer: Calling updateRenderData for tab {actual_tab_id}, markdown length={len(markdown_content)}")
             tab.run_js(f"window.updateRenderData({json.dumps(render_data)})")
 
             # Wait for event-driven finish
-            self._wait_for_render_finished(tab, timeout=5.0)
+            self._wait_for_render_finished(tab, timeout=12.0, context=f"render:{actual_tab_id}")
             
             # Dynamic Resize
             scroll_height = tab.run_js('return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);')

hyw_core/browser_control/service.py

@@ -13,6 +13,10 @@ from typing import Optional, Dict, Any, List
 from loguru import logger
 import trafilatura
 
+# Import intelligent completeness checker
+from ..crawling.completeness import CompletenessChecker, trigger_lazy_load
+from ..crawling.models import CrawlConfig
+
 class ScreenshotService:
     """
     Browser Service using DrissionPage.
@@ -537,12 +541,26 @@ class ScreenshotService:
         """Synchronous screenshot."""
         if not url: return None
         tab = None
+        capture_width = 1024  # Standard capture width
+        
         try:
             self._ensure_ready()
             page = self._manager.page
             if not page: return None
             
-            tab = page.new_tab(url)
+            # Create blank tab first
+            tab = page.new_tab()
+            
+            # Set viewport BEFORE navigation so page renders at target width from the start
+            # This eliminates the need for post-load resize and reflow
+            try:
+                tab.run_cdp('Emulation.setDeviceMetricsOverride', 
+                            width=capture_width, height=900, deviceScaleFactor=1, mobile=False)
+            except:
+                pass
+            
+            # Now navigate to the URL - page will render at target width
+            tab.get(url)
             
             # Start monitoring network traffic
             try:
@@ -552,6 +570,16 @@ class ScreenshotService:
             except Exception as e:
                 logger.warning(f"ScreenshotService: Failed to start network listener: {e}")
 
+            # Initialize crawl config for completeness checking (defined outside try for scope)
+            crawl_config = CrawlConfig(
+                scan_full_page=True,
+                scroll_step=800,
+                scroll_delay=0.5,
+                scroll_timeout=min(timeout, 10),
+                image_load_timeout=8.0,
+                image_stability_checks=3,
+            )
+
             try:
                 # Wait for full page load (including JS execution)
                 tab.wait.load_complete(timeout=timeout)
@@ -676,49 +704,33 @@ class ScreenshotService:
                         f.write(tab.html)
                 except: pass
                 
-                # Use faster scroll step (800) to ensure lazy loaded images appear
-                self._scroll_to_bottom(tab, step=800, delay=2.0, timeout=min(timeout, 10))
+                # Use crawling module for lazy loading trigger (config defined above)
+                trigger_lazy_load(tab, crawl_config)
                 
             except:
                 pass
             
-            # Refine calculation: Set viewport width to 1024
-
-            capture_width = 1024
-            
-            # Calculate actual content height after lazy loading
-            try:
-                # Use a robust height calculation
-                content_height = tab.run_js('''
-                    return Math.max(
-                        document.body.scrollHeight || 0,
-                        document.documentElement.scrollHeight || 0,
-                        document.body.offsetHeight || 0,
-                        document.documentElement.offsetHeight || 0,
-                        document.documentElement.clientHeight || 0
-                    );
-                ''')
-                # Add a small buffer and cap at 15000px to prevent memory issues
-                h = min(int(content_height) + 50, 15000)
-            except:
-                h = 1000  # Fallback
-            
-            # Set viewport to full content size for single-shot capture
-            try:
-                tab.run_cdp('Emulation.setDeviceMetricsOverride', 
-                            width=capture_width, height=h, deviceScaleFactor=1, mobile=False)
-            except:
-                pass
-
-            # Scrollbar Hiding
+            # Scrollbar Hiding first (before any height calculation)
             from .manager import SharedBrowserManager
             SharedBrowserManager.hide_scrollbars(tab)
             
-            # Scroll back to top before screenshot
+            # Scroll back to top
             tab.run_js("window.scrollTo(0, 0);")
             
-            # Content is already loaded by _scroll_to_bottom which waits for images
-            # Just recalculate final height (content may have grown during scrolling)
+            # Use CompletenessChecker to verify all images are loaded
+            checker = CompletenessChecker(crawl_config)
+            completeness = checker.wait_for_complete(tab, timeout=crawl_config.image_load_timeout)
+            
+            logger.info(
+                f"ScreenshotService: Image completeness: "
+                f"{completeness.loaded_images}/{completeness.total_images} loaded, "
+                f"{completeness.failed_images} pending, "
+                f"{completeness.placeholder_images} placeholders, "
+                f"complete={completeness.is_complete}"
+            )
+            
+            # Now calculate final height ONCE after all content loaded
+            # CompletenessChecker already verified height stability
             try:
                 final_height = tab.run_js('''
                     return Math.max(
@@ -728,13 +740,15 @@ class ScreenshotService:
                         document.documentElement.offsetHeight || 0
                     );
                 ''')
-                final_h = min(int(final_height) + 50, 15000)
-                if final_h != h:
-                    tab.run_cdp('Emulation.setDeviceMetricsOverride', 
-                                width=capture_width, height=final_h, deviceScaleFactor=1, mobile=False)
+                h = min(int(final_height) + 50, 15000)
+                tab.run_cdp('Emulation.setDeviceMetricsOverride', 
+                            width=capture_width, height=h, deviceScaleFactor=1, mobile=False)
             except:
                 pass
             
+            # Final scroll to top
+            tab.run_js("window.scrollTo(0, 0);")
+            
             # Use full_page=False because we manually set the viewport to the full height
             # This avoids stitching artifacts and blank spaces
             return tab.get_screenshot(as_base64='jpg', full_page=False)

hyw_core/crawling/__init__.py

@@ -0,0 +1,18 @@
+"""
+hyw_core.crawling - Intelligent Web Crawling Module
+
+Provides Crawl4AI-inspired adaptive crawling with:
+- Page completeness guarantees (image loading verification)
+- Content quality scoring
+- Adaptive stop logic
+"""
+
+from .models import CrawlConfig, PageResult, CompletenessResult
+from .completeness import CompletenessChecker
+
+__all__ = [
+    "CrawlConfig",
+    "PageResult", 
+    "CompletenessResult",
+    "CompletenessChecker",
+]

hyw_core/crawling/completeness.py

@@ -0,0 +1,348 @@
+"""
+Page Completeness Checker
+
+Multi-signal page readiness detection with focus on image loading guarantees.
+Implements Crawl4AI's wait_for_images and networkidle concepts.
+"""
+
+import time
+from typing import Any, Optional, Dict
+from dataclasses import dataclass
+from loguru import logger
+
+from .models import CrawlConfig, CompletenessResult
+
+
+# CSP-compliant JavaScript for image loading verification
+# Based on Crawl4AI's wait_for_images logic
+IMAGE_CHECK_JS = """
+(() => {
+    const results = {
+        total: 0,
+        loaded: 0,
+        failed: 0,
+        placeholders: 0,
+        details: []
+    };
+    
+    const imgs = Array.from(document.querySelectorAll('img'));
+    const viewportHeight = window.innerHeight;
+    const minSize = %d;  // Injected from Python
+    
+    for (const img of imgs) {
+        // Skip tiny images (icons, tracking pixels)
+        if (img.clientWidth < minSize && img.clientHeight < minSize) {
+            continue;
+        }
+        
+        // Skip images outside viewport (unless scan_full_page)
+        const rect = img.getBoundingClientRect();
+        const inViewport = rect.top < viewportHeight * 2 && rect.bottom > -viewportHeight;
+        
+        if (!inViewport && !%s) {  // scan_full_page injected
+            continue;
+        }
+        
+        results.total++;
+        
+        const src = img.src || img.getAttribute('data-src') || '';
+        const isPlaceholder = (
+            (img.getAttribute('data-src') && img.src !== img.getAttribute('data-src')) ||
+            (img.naturalWidth < 50 && img.clientWidth > 100) ||
+            src.includes('placeholder') ||
+            src.includes('loading') ||
+            src.startsWith('data:image/svg+xml') ||
+            (img.naturalWidth === 1 && img.naturalHeight === 1)
+        );
+        
+        if (isPlaceholder) {
+            results.placeholders++;
+            results.details.push({
+                src: src.substring(0, 100),
+                status: 'placeholder',
+                natural: [img.naturalWidth, img.naturalHeight],
+                display: [img.clientWidth, img.clientHeight]
+            });
+            continue;
+        }
+        
+        // Check if fully loaded
+        if (img.complete && img.naturalWidth > 0 && img.naturalHeight > 0) {
+            results.loaded++;
+            results.details.push({
+                src: src.substring(0, 100),
+                status: 'loaded',
+                natural: [img.naturalWidth, img.naturalHeight]
+            });
+        } else {
+            results.failed++;
+            results.details.push({
+                src: src.substring(0, 100),
+                status: 'pending',
+                complete: img.complete,
+                natural: [img.naturalWidth, img.naturalHeight]
+            });
+        }
+    }
+    
+    return results;
+})()
+"""
+
+# JavaScript for height stability check
+HEIGHT_CHECK_JS = """
+(() => {
+    return {
+        height: Math.max(
+            document.body.scrollHeight || 0,
+            document.documentElement.scrollHeight || 0,
+            document.body.offsetHeight || 0,
+            document.documentElement.offsetHeight || 0
+        ),
+        ready: document.readyState === 'complete'
+    };
+})()
+"""
+
+
+class CompletenessChecker:
+    """
+    Multi-signal page completeness verification.
+    
+    Signals checked:
+    1. Image Loading (naturalWidth > 0, not placeholder)
+    2. Height Stability (no layout shifts)
+    3. DOM Ready State
+    
+    Usage:
+        checker = CompletenessChecker(CrawlConfig())
+        result = checker.check(tab)
+        
+        # Or wait until complete
+        result = checker.wait_for_complete(tab)
+    """
+    
+    def __init__(self, config: Optional[CrawlConfig] = None):
+        self._config = config or CrawlConfig()
+    
+    def check(self, tab: Any) -> CompletenessResult:
+        """
+        Perform a single completeness check.
+        
+        Args:
+            tab: DrissionPage tab object
+            
+        Returns:
+            CompletenessResult with all signals
+        """
+        start = time.time()
+        
+        # Check images
+        img_result = self._check_images(tab)
+        
+        # Check height
+        height_result = self._check_height(tab)
+        
+        # Determine overall completeness
+        # Complete if: all real images loaded AND height is stable
+        total_pending = img_result.get('failed', 0) + img_result.get('placeholders', 0)
+        all_images_loaded = total_pending == 0 or img_result.get('total', 0) == 0
+        
+        return CompletenessResult(
+            is_complete=all_images_loaded and height_result.get('ready', False),
+            total_images=img_result.get('total', 0),
+            loaded_images=img_result.get('loaded', 0),
+            failed_images=img_result.get('failed', 0),
+            placeholder_images=img_result.get('placeholders', 0),
+            height=height_result.get('height', 0),
+            height_stable=True,  # Single check can't determine stability
+            network_idle=True,   # TODO: network monitoring
+            check_duration=time.time() - start
+        )
+    
+    def wait_for_complete(
+        self, 
+        tab: Any, 
+        timeout: Optional[float] = None
+    ) -> CompletenessResult:
+        """
+        Wait for page to be fully loaded with image guarantees.
+        
+        Uses multi-signal approach:
+        1. Poll image loading status
+        2. Track height stability
+        3. Respect timeout
+        
+        Args:
+            tab: DrissionPage tab object
+            timeout: Max wait time (default from config)
+            
+        Returns:
+            CompletenessResult (may not be complete if timeout)
+        """
+        timeout = timeout or self._config.image_load_timeout
+        start = time.time()
+        
+        height_history = []
+        stable_count = 0
+        last_result = None
+        
+        logger.debug(f"CompletenessChecker: Starting wait (timeout={timeout}s)")
+        
+        while time.time() - start < timeout:
+            result = self.check(tab)
+            last_result = result
+            
+            # Track height stability
+            height_history.append(result.height)
+            if len(height_history) > self._config.height_stability_checks:
+                height_history.pop(0)
+            
+            # Check if height is stable
+            height_stable = False
+            if len(height_history) >= self._config.height_stability_checks:
+                max_h = max(height_history)
+                min_h = min(height_history)
+                height_stable = (max_h - min_h) <= self._config.height_stability_threshold
+            
+            # Log progress
+            elapsed = time.time() - start
+            logger.debug(
+                f"CompletenessChecker: [{elapsed:.1f}s] "
+                f"images={result.loaded_images}/{result.total_images} "
+                f"pending={result.failed_images} "
+                f"placeholders={result.placeholder_images} "
+                f"height={result.height} stable={height_stable}"
+            )
+            
+            # Check if all images loaded AND height stable
+            all_loaded = (
+                result.failed_images == 0 and 
+                result.placeholder_images == 0
+            )
+            
+            if all_loaded and height_stable:
+                stable_count += 1
+                if stable_count >= self._config.image_stability_checks:
+                    logger.info(
+                        f"CompletenessChecker: Page complete! "
+                        f"{result.loaded_images} images loaded, "
+                        f"height={result.height}, "
+                        f"took {elapsed:.2f}s"
+                    )
+                    return CompletenessResult(
+                        is_complete=True,
+                        total_images=result.total_images,
+                        loaded_images=result.loaded_images,
+                        failed_images=0,
+                        placeholder_images=0,
+                        height=result.height,
+                        height_stable=True,
+                        network_idle=True,
+                        check_duration=elapsed
+                    )
+            else:
+                stable_count = 0
+            
+            # Wait before next check
+            time.sleep(self._config.image_check_interval)
+        
+        # Timeout reached
+        elapsed = time.time() - start
+        logger.warning(
+            f"CompletenessChecker: Timeout after {elapsed:.1f}s! "
+            f"images={last_result.loaded_images}/{last_result.total_images} "
+            f"pending={last_result.failed_images}"
+        )
+        
+        # Return last result with is_complete=False
+        if last_result:
+            return CompletenessResult(
+                is_complete=False,
+                total_images=last_result.total_images,
+                loaded_images=last_result.loaded_images,
+                failed_images=last_result.failed_images,
+                placeholder_images=last_result.placeholder_images,
+                height=last_result.height,
+                height_stable=len(set(height_history)) == 1,
+                network_idle=True,
+                check_duration=elapsed
+            )
+        
+        return CompletenessResult(
+            is_complete=False,
+            total_images=0, loaded_images=0, failed_images=0, placeholder_images=0,
+            height=0, height_stable=False, network_idle=False,
+            check_duration=elapsed
+        )
+    
+    def _check_images(self, tab: Any) -> Dict[str, Any]:
+        """Run image check JavaScript and return results."""
+        try:
+            js = IMAGE_CHECK_JS % (
+                self._config.min_image_size,
+                'true' if self._config.scan_full_page else 'false'
+            )
+            result = tab.run_js(js, as_expr=True)
+            return result or {'total': 0, 'loaded': 0, 'failed': 0, 'placeholders': 0}
+        except Exception as e:
+            logger.warning(f"CompletenessChecker: Image check failed: {e}")
+            return {'total': 0, 'loaded': 0, 'failed': 0, 'placeholders': 0}
+    
+    def _check_height(self, tab: Any) -> Dict[str, Any]:
+        """Run height check JavaScript and return results."""
+        try:
+            result = tab.run_js(HEIGHT_CHECK_JS, as_expr=True)
+            return result or {'height': 0, 'ready': False}
+        except Exception as e:
+            logger.warning(f"CompletenessChecker: Height check failed: {e}")
+            return {'height': 0, 'ready': False}
+
+
+def trigger_lazy_load(tab: Any, config: Optional[CrawlConfig] = None) -> None:
+    """
+    Scroll through page to trigger lazy-loaded images.
+    
+    Implements Crawl4AI's scan_full_page behavior.
+    
+    Args:
+        tab: DrissionPage tab object
+        config: Crawl configuration
+    """
+    config = config or CrawlConfig()
+    if not config.scan_full_page:
+        return
+    
+    start = time.time()
+    current_pos = 0
+    
+    logger.debug("CompletenessChecker: Starting lazy load trigger scroll")
+    
+    try:
+        while time.time() - start < config.scroll_timeout:
+            # Scroll down
+            current_pos += config.scroll_step
+            tab.run_js(f"window.scrollTo(0, {current_pos});")
+            
+            # Wait for lazy load triggers
+            time.sleep(config.scroll_delay)
+            
+            # Check if reached bottom
+            height = tab.run_js("""
+                return Math.max(
+                    document.body.scrollHeight || 0,
+                    document.documentElement.scrollHeight || 0
+                );
+            """, as_expr=True) or 0
+            
+            if current_pos >= height:
+                break
+        
+        # Scroll back to top
+        tab.run_js("window.scrollTo(0, 0);")
+        
+        elapsed = time.time() - start
+        logger.debug(f"CompletenessChecker: Lazy load scroll complete in {elapsed:.1f}s")
+        
+    except Exception as e:
+        logger.warning(f"CompletenessChecker: Lazy load scroll failed: {e}")