entari-plugin-hyw 0.3.5__py3-none-any.whl → 4.0.0rc14__py3-none-any.whl

hyw_core/crawling/completeness.py

@@ -0,0 +1,437 @@
+"""
+Page Completeness Checker
+
+Multi-signal page readiness detection with focus on image loading guarantees.
+Implements Crawl4AI's wait_for_images and networkidle concepts.
+"""
+
+import time
+from typing import Any, Optional, Dict
+from dataclasses import dataclass
+from loguru import logger
+
+from .models import CrawlConfig, CompletenessResult
+
+
+# CSP-compliant JavaScript for image loading verification
+# Based on Crawl4AI's wait_for_images logic
+IMAGE_CHECK_JS = """
+(() => {
+    const results = {
+        total: 0,
+        loaded: 0,
+        failed: 0,
+        placeholders: 0,
+        details: []
+    };
+    
+    const imgs = Array.from(document.querySelectorAll('img'));
+    const viewportHeight = window.innerHeight;
+    const minSize = %d;  // Injected from Python
+    
+    for (const img of imgs) {
+        // Skip tiny images (icons, tracking pixels)
+        if (img.clientWidth < minSize && img.clientHeight < minSize) {
+            continue;
+        }
+        
+        // Skip images outside viewport (unless scan_full_page)
+        const rect = img.getBoundingClientRect();
+        const inViewport = rect.top < viewportHeight * 2 && rect.bottom > -viewportHeight;
+        
+        if (!inViewport && !%s) {  // scan_full_page injected
+            continue;
+        }
+        
+        results.total++;
+        
+        const src = img.src || img.getAttribute('data-src') || '';
+        const dataSrc = img.getAttribute('data-src') || img.getAttribute('data-original') || 
+                        img.getAttribute('data-lazy-src') || img.getAttribute('data-lazy') || '';
+        const className = (typeof img.className === 'string' ? img.className : '').toLowerCase();
+        const loadingAttr = img.getAttribute('loading') || '';
+        
+        // Enhanced placeholder detection for blurred preview images (like mcmod.cn)
+        const isPlaceholder = (
+            // 1. data-src exists but not yet loaded into src
+            (dataSrc && img.src !== dataSrc) ||
+            // 2. Natural size much smaller than display size (blurred placeholder)
+            (img.naturalWidth < 50 && img.clientWidth > 100) ||
+            (img.naturalWidth < 100 && img.clientWidth > 200 && img.naturalWidth * 4 < img.clientWidth) ||
+            // 3. Common placeholder keywords in src
+            src.includes('placeholder') ||
+            src.includes('loading') ||
+            src.includes('blank') ||
+            // 4. SVG placeholder or 1x1 tracking pixel
+            src.startsWith('data:image/svg+xml') ||
+            (img.naturalWidth === 1 && img.naturalHeight === 1) ||
+            // 5. Lazy-loading class indicators (common patterns)
+            className.includes('lazy') ||
+            className.includes('lazyload') ||
+            className.includes('lozad') ||
+            className.includes('b-lazy') ||
+            // 6. Blur indicators (common for LQIP - Low Quality Image Placeholder)
+            className.includes('blur') ||
+            src.includes('blur') ||
+            src.includes('thumb') ||
+            src.includes('thumbnail') ||
+            // 7. loading="lazy" + not complete (browser native lazy loading)
+            (loadingAttr === 'lazy' && !img.complete) ||
+            // 8. CSS blur filter applied (visual blurring)
+            (window.getComputedStyle(img).filter || '').includes('blur')
+        );
+        
+        if (isPlaceholder) {
+            results.placeholders++;
+            results.details.push({
+                src: src.substring(0, 100),
+                status: 'placeholder',
+                natural: [img.naturalWidth, img.naturalHeight],
+                display: [img.clientWidth, img.clientHeight]
+            });
+            continue;
+        }
+        
+        // Check if fully loaded
+        if (img.complete && img.naturalWidth > 0 && img.naturalHeight > 0) {
+            results.loaded++;
+            results.details.push({
+                src: src.substring(0, 100),
+                status: 'loaded',
+                natural: [img.naturalWidth, img.naturalHeight]
+            });
+        } else {
+            results.failed++;
+            results.details.push({
+                src: src.substring(0, 100),
+                status: 'pending',
+                complete: img.complete,
+                natural: [img.naturalWidth, img.naturalHeight]
+            });
+        }
+    }
+    
+    return results;
+})()
+"""
+
+# JavaScript for height stability check
+HEIGHT_CHECK_JS = """
+(() => {
+    return {
+        height: Math.max(
+            document.body.scrollHeight || 0,
+            document.documentElement.scrollHeight || 0,
+            document.body.offsetHeight || 0,
+            document.documentElement.offsetHeight || 0
+        ),
+        ready: document.readyState === 'complete'
+    };
+})()
+"""
+
+
+class CompletenessChecker:
+    """
+    Multi-signal page completeness verification.
+    
+    Signals checked:
+    1. Image Loading (naturalWidth > 0, not placeholder)
+    2. Height Stability (no layout shifts)
+    3. DOM Ready State
+    
+    Usage:
+        checker = CompletenessChecker(CrawlConfig())
+        result = checker.check(tab)
+        
+        # Or wait until complete
+        result = checker.wait_for_complete(tab)
+    """
+    
+    def __init__(self, config: Optional[CrawlConfig] = None):
+        self._config = config or CrawlConfig()
+    
+    def check(self, tab: Any) -> CompletenessResult:
+        """
+        Perform a single completeness check.
+        
+        Args:
+            tab: DrissionPage tab object
+            
+        Returns:
+            CompletenessResult with all signals
+        """
+        start = time.time()
+        
+        # Check images
+        img_result = self._check_images(tab)
+        
+        # Check height
+        height_result = self._check_height(tab)
+        
+        # Determine overall completeness
+        # Complete if: all real images loaded AND height is stable
+        total_pending = img_result.get('failed', 0) + img_result.get('placeholders', 0)
+        all_images_loaded = total_pending == 0 or img_result.get('total', 0) == 0
+        
+        return CompletenessResult(
+            is_complete=all_images_loaded and height_result.get('ready', False),
+            total_images=img_result.get('total', 0),
+            loaded_images=img_result.get('loaded', 0),
+            failed_images=img_result.get('failed', 0),
+            placeholder_images=img_result.get('placeholders', 0),
+            height=height_result.get('height', 0),
+            height_stable=True,  # Single check can't determine stability
+            network_idle=True,   # TODO: network monitoring
+            check_duration=time.time() - start
+        )
+    
+    def wait_for_complete(
+        self, 
+        tab: Any, 
+        timeout: Optional[float] = None
+    ) -> CompletenessResult:
+        """
+        Wait for page to be fully loaded with image guarantees.
+        
+        Uses multi-signal approach:
+        1. Poll image loading status
+        2. Track height stability
+        3. Respect timeout
+        
+        Args:
+            tab: DrissionPage tab object
+            timeout: Max wait time (default from config)
+            
+        Returns:
+            CompletenessResult (may not be complete if timeout)
+        """
+        timeout = timeout or self._config.image_load_timeout
+        start = time.time()
+        
+        height_history = []
+        stable_count = 0
+        last_result = None
+        
+        logger.debug(f"CompletenessChecker: Starting wait (timeout={timeout}s)")
+        
+        while time.time() - start < timeout:
+            result = self.check(tab)
+            last_result = result
+            
+            # Track height stability
+            height_history.append(result.height)
+            if len(height_history) > self._config.height_stability_checks:
+                height_history.pop(0)
+            
+            # Check if height is stable
+            height_stable = False
+            if len(height_history) >= self._config.height_stability_checks:
+                max_h = max(height_history)
+                min_h = min(height_history)
+                height_stable = (max_h - min_h) <= self._config.height_stability_threshold
+            
+            # Log progress
+            elapsed = time.time() - start
+            logger.debug(
+                f"CompletenessChecker: [{elapsed:.1f}s] "
+                f"images={result.loaded_images}/{result.total_images} "
+                f"pending={result.failed_images} "
+                f"placeholders={result.placeholder_images} "
+                f"height={result.height} stable={height_stable}"
+            )
+            
+            # Check if all images loaded AND height stable
+            all_loaded = (
+                result.failed_images == 0 and 
+                result.placeholder_images == 0
+            )
+            
+            if all_loaded and height_stable:
+                stable_count += 1
+                if stable_count >= self._config.image_stability_checks:
+                    logger.info(
+                        f"CompletenessChecker: Page complete! "
+                        f"{result.loaded_images} images loaded, "
+                        f"height={result.height}, "
+                        f"took {elapsed:.2f}s"
+                    )
+                    return CompletenessResult(
+                        is_complete=True,
+                        total_images=result.total_images,
+                        loaded_images=result.loaded_images,
+                        failed_images=0,
+                        placeholder_images=0,
+                        height=result.height,
+                        height_stable=True,
+                        network_idle=True,
+                        check_duration=elapsed
+                    )
+            else:
+                stable_count = 0
+            
+            # Wait before next check
+            time.sleep(self._config.image_check_interval)
+        
+        # Timeout reached
+        elapsed = time.time() - start
+        logger.warning(
+            f"CompletenessChecker: Timeout after {elapsed:.1f}s! "
+            f"images={last_result.loaded_images}/{last_result.total_images} "
+            f"pending={last_result.failed_images}"
+        )
+        
+        # Return last result with is_complete=False
+        if last_result:
+            return CompletenessResult(
+                is_complete=False,
+                total_images=last_result.total_images,
+                loaded_images=last_result.loaded_images,
+                failed_images=last_result.failed_images,
+                placeholder_images=last_result.placeholder_images,
+                height=last_result.height,
+                height_stable=len(set(height_history)) == 1,
+                network_idle=True,
+                check_duration=elapsed
+            )
+        
+        return CompletenessResult(
+            is_complete=False,
+            total_images=0, loaded_images=0, failed_images=0, placeholder_images=0,
+            height=0, height_stable=False, network_idle=False,
+            check_duration=elapsed
+        )
+    
+    def _check_images(self, tab: Any) -> Dict[str, Any]:
+        """Run image check JavaScript and return results."""
+        try:
+            js = IMAGE_CHECK_JS % (
+                self._config.min_image_size,
+                'true' if self._config.scan_full_page else 'false'
+            )
+            result = tab.run_js(js, as_expr=True)
+            return result or {'total': 0, 'loaded': 0, 'failed': 0, 'placeholders': 0}
+        except Exception as e:
+            logger.warning(f"CompletenessChecker: Image check failed: {e}")
+            return {'total': 0, 'loaded': 0, 'failed': 0, 'placeholders': 0}
+    
+    def _check_height(self, tab: Any) -> Dict[str, Any]:
+        """Run height check JavaScript and return results."""
+        try:
+            result = tab.run_js(HEIGHT_CHECK_JS, as_expr=True)
+            return result or {'height': 0, 'ready': False}
+        except Exception as e:
+            logger.warning(f"CompletenessChecker: Height check failed: {e}")
+            return {'height': 0, 'ready': False}
+
+
+def trigger_lazy_load(tab: Any, config: Optional[CrawlConfig] = None) -> None:
+    """
+    Scroll through page to trigger lazy-loaded images.
+    
+    Implements Crawl4AI's scan_full_page behavior.
+    Strategy: Fast scroll with minimal delay (0.2s) per step to trigger network requests,
+    then wait at the bottom for all images to settle.
+    
+    Args:
+        tab: DrissionPage tab object
+        config: Crawl configuration
+    """
+    config = config or CrawlConfig()
+    if not config.scan_full_page:
+        return
+    
+    start = time.time()
+    current_pos = 0
+    
+    logger.info(f"CompletenessChecker: Starting lazy load scroll (fast scroll + final wait)")
+    
+    try:
+        max_scroll_steps = 100
+        step_count = 0
+        
+        # 1. Fast Scroll Phase
+        while step_count < max_scroll_steps:
+            step_count += 1
+            current_pos += config.scroll_step
+            tab.run_js(f"window.scrollTo(0, {current_pos});")
+            
+            # Simple fixed delay per step (0.2s) as requested
+            time.sleep(0.2)
+            
+            # Check if reached bottom
+            height = tab.run_js("""
+                Math.max(
+                    document.body.scrollHeight || 0,
+                    document.documentElement.scrollHeight || 0
+                )
+            """, as_expr=True) or 0
+            
+            if current_pos >= height:
+                logger.debug(f"CompletenessChecker: Reached bottom at position {current_pos}")
+                break
+
+        # 2. Wait Phase at Bottom (Wait for images to settle - reduced timeout)
+        logger.debug("CompletenessChecker: Reached bottom, waiting for images to settle (max 2s)...")
+        wait_start = time.time()
+        max_wait_at_bottom = 2.0  # Reduced from 8s to 2s - scroll usually triggers loading quickly
+        
+        # Quick check: just verify images are not placeholders (simplified check)
+        check_all_images_js = """
+        (() => {
+            const imgs = Array.from(document.querySelectorAll('img'));
+            if (imgs.length === 0) return true;
+            
+            // Quick check: count non-placeholder images that are loaded
+            let loaded_count = 0;
+            let total_count = 0;
+            
+            for (const img of imgs) {
+                // Skip tiny images
+                if (img.clientWidth < 50 && img.clientHeight < 50) continue;
+                
+                total_count++;
+                const dataSrc = img.getAttribute('data-src') || img.getAttribute('data-original') || '';
+                const src = img.src || '';
+                
+                // Check if placeholder
+                const isPlaceholder = (
+                    (dataSrc && img.src !== dataSrc) ||
+                    (img.naturalWidth < 50 && img.clientWidth > 100) ||
+                    src.includes('placeholder') || src.includes('loading')
+                );
+                
+                // If not placeholder and loaded, count it
+                if (!isPlaceholder && img.complete && img.naturalWidth > 0) {
+                    loaded_count++;
+                }
+            }
+            
+            // If most images are loaded (80%+), consider it done
+            return total_count === 0 || (loaded_count / total_count) >= 0.8;
+        })()
+        """
+
+        # Quick check loop with shorter interval
+        check_count = 0
+        max_checks = 4  # 2s / 0.5s = 4 checks max
+        while check_count < max_checks:
+            try:
+                all_loaded = tab.run_js(check_all_images_js, as_expr=True)
+                if all_loaded:
+                    elapsed_wait = time.time() - wait_start
+                    logger.debug(f"CompletenessChecker: Images settled at bottom in {elapsed_wait:.1f}s")
+                    break
+            except:
+                pass
+            time.sleep(0.5)
+            check_count += 1
+        
+        # Scroll back to top
+        tab.run_js("window.scrollTo(0, 0);")
+        
+        elapsed = time.time() - start
+        logger.info(f"CompletenessChecker: Lazy load scroll complete - {step_count} steps in {elapsed:.1f}s")
+        
+    except Exception as e:
+        logger.warning(f"CompletenessChecker: Lazy load scroll failed: {e}")

hyw_core/crawling/models.py

@@ -0,0 +1,88 @@
+"""
+Crawling Data Models
+
+Core data structures for the intelligent crawling system.
+"""
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Set
+
+
+@dataclass
+class CrawlConfig:
+    """
+    Crawling configuration with Crawl4AI-style parameters.
+    
+    Focuses on page completeness and image loading guarantees.
+    """
+    # Wait Strategy (Priority: Image Loading)
+    wait_for_images: bool = True
+    wait_until: str = "networkidle"  # domcontentloaded | networkidle | load
+    delay_before_return: float = 0.1
+    page_timeout: float = 30.0
+    
+    # Image Loading Specific
+    image_load_timeout: float = 10.0  # Max wait for images
+    image_stability_checks: int = 3   # Consecutive stable checks needed
+    image_check_interval: float = 0.2 # Interval between checks
+    min_image_size: int = 50          # Ignore images smaller than this
+    
+    # Scroll for lazy loading
+    scan_full_page: bool = True
+    scroll_step: int = 800
+    scroll_delay: float = 0.5
+    scroll_timeout: float = 15.0
+    
+    # Height Stability
+    height_stability_checks: int = 3
+    height_stability_threshold: int = 10  # pixels
+    
+    # Future: Adaptive Stop Logic
+    confidence_threshold: float = 0.75
+    min_gain_threshold: float = 0.1
+    max_pages: int = 20
+
+
+@dataclass
+class CompletenessResult:
+    """Result from completeness check."""
+    is_complete: bool
+    total_images: int
+    loaded_images: int
+    failed_images: int
+    placeholder_images: int
+    height: int
+    height_stable: bool
+    network_idle: bool
+    check_duration: float
+    
+    @property
+    def image_load_ratio(self) -> float:
+        if self.total_images == 0:
+            return 1.0
+        return self.loaded_images / self.total_images
+
+
+@dataclass
+class PageResult:
+    """Result from fetching a single page."""
+    url: str
+    final_url: str
+    title: str
+    html: str
+    content: str  # Extracted markdown
+    images: List[str] = field(default_factory=list)  # base64 images
+    screenshot: Optional[str] = None
+    
+    # Quality Signals
+    load_time: float = 0.0
+    completeness: Optional[CompletenessResult] = None
+    
+    # Error handling
+    error: Optional[str] = None
+    
+    @property
+    def is_complete(self) -> bool:
+        if self.completeness is None:
+            return False
+        return self.completeness.is_complete

hyw_core/definitions.py

@@ -0,0 +1,104 @@
+"""
+Centralized Definitions
+
+All global prompts and tool definitions for the pipeline stages.
+"""
+
+from typing import Dict, Any
+
+# Used by SummaryStage - language appended at runtime
+SUMMARY_REPORT_SP = """# 你是一个总结助手 (Agent), 你的职责是基于搜索工具给出的信息，回答用户的问题或解释用户问题中的关键词。
+## 核心原则
+最小限度使用自身知识, 尽可能使用 web_tool 获取信息.
+
+## 工具使用指南
+- 适当时候调用 `refuse_answer`
+
+## 回答格式
+- `# ` 大标题约 8-10 个字
+- <summary>...</summary> 约 100 字的概括
+- 二级标题 + markdown 正文
+- 正文使用 [1] 格式引用信息来源, 无需写出源, 系统自动渲染
+"""
+
+def get_refuse_answer_tool() -> Dict[str, Any]:
+    """Tool for refusing to answer inappropriate content."""
+    return {
+        "type": "function",
+        "function": {
+            "name": "refuse_answer",
+            "description": "违规内容拒绝回答，内容涉及隐喻政治事件、任务、现役国家领导人、r18+、r18g(但不包含正常galgame、科普等)",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "reason": {"type": "string", "description": "拒绝回答的原因（展示给用户）"},
+                },
+                "required": ["reason"],
+            },
+        },
+    }
+
+
+def get_web_tool() -> Dict[str, Any]:
+    """Tool for web search with filter syntax and URL screenshot."""
+    return {
+        "type": "function",
+        "function": {
+            "name": "web_tool",
+            "description": """搜索网页或截图指定URL。用于获取最新信息、查找资料。
+## 使用方式
+网页搜索(大部分问题优先使用此方法): 
+直接传入搜索词如 "python async" 会返回搜索结果列表
+
+网页截图(当用户明确要求截图时使用):
+传入完整URL如 "https://example.com" 会直接截图该页面
+
+网页搜索 + 网页截图(可以预测能直接搜到什么样的结果时使用): (最终截图最多3张)
+- 域名过滤: "github=2: python async" → 会搜索 "python async github" 并截图 链接/标题包含 "github" 的前2个结果
+- 序号选择: "1,2: minecraft mods" → 会搜索 "minecraft mods" 并截图第1、2个结果
+- 多域名: "mcmod=1, github=1: forge mod" → 会搜索 "forge mod mcmod github" 并截图 链接/标题包含 "mcmod" 的前1个结果和 链接/标题包含 "github" 的前1个结果
+""",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "搜索查询或URL。支持过滤器语法(见描述)"
+                    }
+                },
+                "required": ["query"]
+            }
+        }
+    }
+
+
+# =============================================================================
+# AGENT PROMPTS
+# =============================================================================
+
+AGENT_SYSTEM_PROMPT = """# 你是一个智能助手 (Agent), 你的职责是使用 `web_tool` 工具来帮助用户搜索网页或截图URL, 同时完成用户分配给你的任务.
+## 任务
+理解用户意图分配给你的任务.
+如果用户没有明确分配任务, 则默认任务为解释用户问题中的关键词.
+
+## 核心原则
+最小限度使用自身知识, 尽可能使用 web_tool 获取信息.
+
+## 工具使用指南
+- 并行调用工具
+    - 网页搜索: 可以同时调用3次, 其中URL截图消耗较大, 最多同时调用1个
+- 积极使用 web_tool 获取信息
+    - 搜索时, 关键词保证简单、指向准确、利于传统搜索引擎.
+    - 获取页面截图时, 只使用官方性较强的 wiki、官方网站、资源站等等, 不使用第三方转载新闻网站.
+- 最多可调用2轮工具, 之后必须给出最终回答
+- 适当时候调用 `refuse_answer`
+- 对于具体任务, 如果是转述、格式化、翻译等, 请直接给出最终回答, 不再调用工具
+
+## 回答格式
+- `# ` 大标题约 8-10 个字
+- <summary>...</summary> 约 100 字的概括
+- 二级标题 + markdown 正文
+- 正文使用 [1] 格式引用信息来源, 无需写出源, 系统自动渲染
+"""
+
+