html-reader-llm 0.0.1__py3-none-any.whl

html_reader_llm/render_detect.py

@@ -0,0 +1,415 @@
+"""Render detection — compare HTTP vs Chrome --dump-dom to decide if browser rendering is needed.
+
+Three detection dimensions:
+- DOM structure diff (node count ratio)
+- Text content similarity (word-set Jaccard)
+- SPA shell detection (empty body heuristic)
+
+Output: RenderDetectResult with per-dimension scores + use_browser decision.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import re
+import subprocess
+from typing import TypedDict
+
+from selectolax.lexbor import LexborHTMLParser
+from trafilatura import fetch_url
+
+from html_reader_llm.browser import get_browser_path
+from html_reader_llm.simplify import _walk_nodes
+
+logger = logging.getLogger(__name__)
+
+# --- ENV helpers ---
+
+
+def _env_int(name: str, default: int) -> int:
+    val = os.environ.get(f"HTMLREADER_{name}")
+    if val is None:
+        return default
+    try:
+        return int(val)
+    except ValueError:
+        return default
+
+
+# --- Defaults ---
+
+HTTP_TIMEOUT = _env_int("HTTP_TIMEOUT", 15)
+CHROME_TIMEOUT = _env_int("CHROME_TIMEOUT", 30)
+CHROME_VIRTUAL_TIME_BUDGET = _env_int("CHROME_VIRTUAL_TIME_BUDGET", 5000)
+
+_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36"
+)
+
+# --- Confidence weights ---
+
+W_DOM_RATIO = 0.3
+W_TEXT_SIMILARITY = 0.4
+W_SPA_SHELL = 0.3
+USE_BROWSER_THRESHOLD = 0.7
+
+# --- Cache dir for HTTP responses ---
+_CACHE_DIR = os.path.join(".temp", "http_cache")
+
+
+# =========================================================================
+# Result type
+# =========================================================================
+
+
+class RenderDetectResult(TypedDict):
+    http_status: int | None
+    http_html: str | None
+    chrome_html: str | None
+    dom_ratio: float
+    text_similarity: float
+    spa_shell: bool
+    confidence: float
+    use_browser: bool
+
+
+# =========================================================================
+# 1. HTTP fetch (trafilatura + file cache)
+# =========================================================================
+
+
+def _cache_path(url: str) -> str:
+    """Compute cache file path for a URL."""
+    url_hash = hashlib.md5(url.encode()).hexdigest()[:12]
+    return os.path.join(_CACHE_DIR, f"{url_hash}.html")
+
+
+def _fetch_http(url: str, use_cache: bool = True) -> tuple[int | None, str | None]:
+    """Fetch URL via trafilatura.fetch_url with file cache.
+
+    Returns (status_code, html_body) or (None, None) on error.
+    trafilatura handles gzip, charset, redirects, SSL internally.
+    """
+    cache_file = _cache_path(url)
+
+    # Try cache first
+    if use_cache and os.path.isfile(cache_file):
+        try:
+            with open(cache_file, encoding="utf-8") as f:
+                html = f.read()
+            # Strip URL comment header if present
+            if html.startswith("<!-- source:"):
+                nl = html.find("\n")
+                if nl != -1:
+                    html = html[nl + 1 :]
+            logger.debug("HTTP fetch %s -> cache hit (%d chars)", url, len(html))
+            return 200, html
+        except OSError:
+            pass
+
+    # Fetch via trafilatura
+    html = fetch_url(url)
+    if not html:
+        logger.debug("HTTP fetch %s -> failed", url)
+        return None, None
+
+    logger.debug("HTTP fetch %s -> %d chars", url, len(html))
+
+    # Save to cache
+    if use_cache:
+        try:
+            os.makedirs(_CACHE_DIR, exist_ok=True)
+            with open(cache_file, "w", encoding="utf-8") as f:
+                f.write(f"<!-- source: {url} -->\n{html}")
+        except OSError as e:
+            logger.debug("Cache write failed: %s", e)
+
+    return 200, html
+
+
+# =========================================================================
+# 2. Chrome dump-dom
+# =========================================================================
+
+
+def _fetch_chrome(
+    url: str,
+    chrome_path: str | None = None,
+    user_data_dir: str | None = None,
+) -> str | None:
+    """Fetch rendered HTML via Chrome --dump-dom.
+
+    Args:
+        url: URL to fetch.
+        chrome_path: Optional Chrome/Edge path. If None, auto-detects.
+        user_data_dir: Optional Chrome user-data-dir for caching. If None, not used.
+
+    Returns rendered HTML or None on error.
+    """
+    if chrome_path is None:
+        chrome_path = get_browser_path()
+    if chrome_path is None:
+        logger.debug("Chrome dump-dom: no browser path found")
+        return None
+
+    cmd = [
+        chrome_path,
+        "--headless",
+        "--disable-gpu",
+        "--no-sandbox",
+        f"--virtual-time-budget={CHROME_VIRTUAL_TIME_BUDGET}",
+        "--blink-settings=imagesEnabled=false",
+        f"--user-agent={_USER_AGENT}",
+    ]
+
+    if user_data_dir:
+        os.makedirs(user_data_dir, exist_ok=True)
+        cmd.append(f"--user-data-dir={user_data_dir}")
+
+    cmd.extend(["--dump-dom", url])
+
+    logger.debug("Chrome dump-dom: %s", " ".join(cmd[:3]) + " ... " + url)
+
+    try:
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            timeout=CHROME_TIMEOUT,
+            text=True,
+            check=False,
+        )
+        if result.returncode != 0:
+            logger.debug(
+                "Chrome dump-dom: exit code %d, stderr: %s",
+                result.returncode,
+                result.stderr[:200] if result.stderr else "",
+            )
+        html = result.stdout
+        if html and len(html) > 100:
+            logger.debug("Chrome dump-dom: got %d chars", len(html))
+            return html
+        logger.debug("Chrome dump-dom: empty or too short output")
+        return None
+    except subprocess.TimeoutExpired:
+        logger.debug("Chrome dump-dom: timeout after %ds", CHROME_TIMEOUT)
+        return None
+    except FileNotFoundError:
+        logger.debug("Chrome dump-dom: binary not found: %s", chrome_path)
+        return None
+
+
+# =========================================================================
+# 3. Detection dimensions
+# =========================================================================
+
+
+def _count_element_nodes(html: str) -> int:
+    """Count all element nodes in HTML."""
+    parser = LexborHTMLParser(html)
+    count = 0
+    for node in _walk_nodes(parser.root):
+        if node.is_element_node:
+            count += 1
+    return count
+
+
+def _dom_ratio(html_a: str, html_b: str) -> float:
+    """Compute DOM node count ratio: min(a,b) / max(a,b).
+
+    Returns 0.0 if either side has 0 nodes.
+    """
+    count_a = _count_element_nodes(html_a)
+    count_b = _count_element_nodes(html_b)
+    if count_a == 0 and count_b == 0:
+        return 1.0
+    if count_a == 0 or count_b == 0:
+        return 0.0
+    return min(count_a, count_b) / max(count_a, count_b)
+
+
+_TAG_RE = re.compile(r"<[^>]+>")
+_WS_RE = re.compile(r"\s+")
+
+
+def _extract_visible_text(html: str) -> str:
+    """Strip HTML tags and collapse whitespace to get visible text."""
+    text = _TAG_RE.sub(" ", html)
+    text = _WS_RE.sub(" ", text).strip()
+    return text
+
+
+def _text_similarity(html_a: str, html_b: str) -> float:
+    """Compute Jaccard similarity of word sets from two HTML bodies.
+
+    For Chinese text, uses character-level tokens. For English, uses word-level.
+    """
+    text_a = _extract_visible_text(html_a)
+    text_b = _extract_visible_text(html_b)
+    if not text_a and not text_b:
+        return 1.0
+    if not text_a or not text_b:
+        return 0.0
+
+    # Tokenize: split on whitespace, then further split Chinese chars
+    words_a = _tokenize(text_a)
+    words_b = _tokenize(text_b)
+
+    if not words_a and not words_b:
+        return 1.0
+    if not words_a or not words_b:
+        return 0.0
+
+    set_a = set(words_a)
+    set_b = set(words_b)
+    intersection = set_a & set_b
+    union = set_a | set_b
+    return len(intersection) / len(union)
+
+
+_CN_CHAR_RE = re.compile(r"[\u4e00-\u9fff\u3400-\u4dbf]")
+
+
+def _tokenize(text: str) -> list[str]:
+    """Tokenize text: English words as-is, Chinese chars individually."""
+    tokens: list[str] = []
+    for word in text.split():
+        if _CN_CHAR_RE.search(word):
+            # Chinese: split into individual characters
+            for ch in word:
+                if _CN_CHAR_RE.match(ch):
+                    tokens.append(ch)
+                else:
+                    tokens.append(ch)
+        else:
+            tokens.append(word.lower())
+    return tokens
+
+
+def _is_spa_shell(html: str) -> bool:
+    """Detect SPA shell: body contains only noscript and/or a single empty div."""
+    parser = LexborHTMLParser(html)
+    body = parser.css_first("body")
+    if body is None:
+        return False
+
+    # Count meaningful child elements of body
+    meaningful = 0
+    has_app_div = False
+    child = body.first_child
+    while child is not None:
+        if child.is_element_node:
+            tag = child.tag.lower()
+            if tag == "noscript":
+                # noscript is expected in SPA shells
+                meaningful += 1
+            elif tag in ("div", "section", "main"):
+                meaningful += 1
+                # Check if it's an app/root div (empty or near-empty)
+                text = child.text(deep=True).strip()
+                if len(text) < 50:
+                    el_id = child.attributes.get("id", "").lower()
+                    el_class = child.attributes.get("class", "").lower()
+                    if (
+                        el_id in ("app", "root", "__next", "__nuxt")
+                        or "app" in el_class
+                    ):
+                        has_app_div = True
+            else:
+                meaningful += 1
+        child = child.next
+
+    # SPA shell: only noscript + one div with app-like id
+    return has_app_div and meaningful <= 3
+
+
+# =========================================================================
+# 4. Confidence + decision
+# =========================================================================
+
+
+def _compute_confidence(
+    dom_ratio: float, text_similarity: float, spa_shell: bool
+) -> float:
+    """Compute weighted confidence score. Lower = more likely needs browser."""
+    spa_score = 1.0 if spa_shell else 0.0
+    # Invert: high ratio/similarity = high confidence no browser needed
+    score = (
+        W_DOM_RATIO * dom_ratio
+        + W_TEXT_SIMILARITY * text_similarity
+        + W_SPA_SHELL * (1.0 - spa_score)  # spa_shell=True means low confidence
+    )
+    return round(score, 4)
+
+
+# =========================================================================
+# 5. Main entry point
+# =========================================================================
+
+
+def detect_render(
+    url: str,
+    chrome_path: str | None = None,
+    user_data_dir: str | None = None,
+    use_cache: bool = True,
+) -> RenderDetectResult:
+    """Compare urllib vs Chrome --dump-dom and decide if browser rendering is needed.
+
+    Args:
+        url: The URL to analyze.
+        chrome_path: Optional Chrome/Edge path. If None, auto-detects.
+        user_data_dir: Optional Chrome user-data-dir for caching. If None, not used.
+        use_cache: Whether to use HTTP response cache.
+
+    Returns:
+        RenderDetectResult with all metrics and use_browser decision.
+    """
+    logger.debug("=== render detection start: %s ===", url)
+
+    # Fetch HTTP
+    http_status, http_html = _fetch_http(url, use_cache=use_cache)
+
+    # Fetch Chrome
+    chrome_html = _fetch_chrome(url, chrome_path, user_data_dir)
+
+    # Compute dimensions
+    if http_html and chrome_html:
+        dom_ratio_val = _dom_ratio(http_html, chrome_html)
+        text_sim_val = _text_similarity(http_html, chrome_html)
+    elif http_html:
+        # Chrome unavailable — can only check SPA shell
+        dom_ratio_val = 0.0
+        text_sim_val = 0.0
+    else:
+        dom_ratio_val = 0.0
+        text_sim_val = 0.0
+
+    spa_shell_val = _is_spa_shell(http_html) if http_html else False
+
+    # Confidence + decision
+    confidence_val = _compute_confidence(dom_ratio_val, text_sim_val, spa_shell_val)
+    use_browser_val = confidence_val < USE_BROWSER_THRESHOLD
+
+    result: RenderDetectResult = {
+        "http_status": http_status,
+        "http_html": http_html,
+        "chrome_html": chrome_html,
+        "dom_ratio": dom_ratio_val,
+        "text_similarity": text_sim_val,
+        "spa_shell": spa_shell_val,
+        "confidence": confidence_val,
+        "use_browser": use_browser_val,
+    }
+
+    logger.debug(
+        "=== render detection done: dom=%.3f text=%.3f spa=%s conf=%.3f use_browser=%s ===",
+        dom_ratio_val,
+        text_sim_val,
+        spa_shell_val,
+        confidence_val,
+        use_browser_val,
+    )
+
+    return result

html_reader_llm/selector_validator.py

@@ -0,0 +1,177 @@
+"""CSS 选择器验证模块
+
+用于验证 CSS 选择器是否能正确提取目标内容
+"""
+
+from typing import Any
+
+from selectolax.lexbor import LexborHTMLParser
+
+
+class SelectorValidator:
+    """CSS 选择器验证器"""
+
+    def __init__(self, html: str):
+        """初始化验证器
+
+        Args:
+            html: HTML 内容
+        """
+        self.parser = LexborHTMLParser(html)
+
+    def validate_selector(
+        self, selector: str, method: str = "$text", expected_count: int | None = None
+    ) -> dict[str, Any]:
+        """验证单个选择器
+
+        Args:
+            selector: CSS 选择器
+            method: 取值方式 ($text, $html, @attr)
+            expected_count: 期望匹配的数量（None 表示不检查）
+
+        Returns:
+            验证结果字典，包含:
+            - valid: bool - 是否有效
+            - count: int - 匹配数量
+            - samples: list - 示例值（最多3个）
+            - error: str | None - 错误信息
+        """
+        result = {"valid": False, "count": 0, "samples": [], "error": None}
+
+        try:
+            nodes = self.parser.css(selector)
+            result["count"] = len(nodes)
+
+            # 提取示例值
+            for node in nodes[:3]:
+                if method.startswith("@"):
+                    # 属性值: @content, @href, @src 等
+                    attr_name = method[1:]
+                    value = node.attributes.get(attr_name, "")
+                elif method == "$html":
+                    value = node.html
+                else:
+                    # 默认 $text
+                    value = node.text()
+
+                result["samples"].append(value[:100] if value else "")
+
+            # 检查是否有效
+            if result["count"] > 0:
+                result["valid"] = True
+
+            # 检查期望数量
+            if expected_count is not None and result["count"] != expected_count:
+                result["valid"] = False
+                result["error"] = (
+                    f"期望 {expected_count} 个匹配，实际 {result['count']} 个"
+                )
+
+        except Exception as e:
+            result["error"] = str(e)
+
+        return result
+
+    def validate_rules(
+        self, rules: dict[str, dict], list_selector: str | None = None
+    ) -> dict[str, dict]:
+        """验证一组规则
+
+        Args:
+            rules: 规则字典，格式: {field_name: {css: str, method: str}}
+            list_selector: 列表选择器（如果 rules 是列表项的字段）
+
+        Returns:
+            验证结果字典，格式: {field_name: validation_result}
+        """
+        results = {}
+
+        if list_selector:
+            # 验证列表选择器
+            list_result = self.validate_selector(list_selector)
+            results["_list"] = list_result
+
+            # 如果列表有效，验证列表项内的字段
+            if list_result["valid"] and list_result["count"] > 0:
+                list_nodes = self.parser.css(list_selector)
+                first_node = list_nodes[0]
+
+                # 创建临时验证器来验证列表项
+                item_html = first_node.html
+                if item_html:
+                    item_validator = SelectorValidator(item_html)
+                    for field_name, rule in rules.items():
+                        results[field_name] = item_validator.validate_selector(
+                            rule["css"], rule.get("method", "$text")
+                        )
+        else:
+            # 直接验证字段
+            for field_name, rule in rules.items():
+                results[field_name] = self.validate_selector(
+                    rule["css"], rule.get("method", "$text")
+                )
+
+        return results
+
+    def print_validation_report(
+        self, results: dict[str, dict], title: str = "验证报告"
+    ) -> None:
+        """打印验证报告
+
+        Args:
+            results: validate_rules 返回的结果
+            title: 报告标题
+        """
+        print(f"\n=== {title} ===\n")
+
+        for field_name, result in results.items():
+            status = "✅" if result["valid"] else "❌"
+            print(f"{status} {field_name}")
+
+            if result["error"]:
+                print(f"   错误: {result['error']}")
+            else:
+                print(f"   匹配数量: {result['count']}")
+                if result["samples"]:
+                    print("   示例值:")
+                    for i, sample in enumerate(result["samples"]):
+                        print(f"     {i + 1}. {sample}...")
+
+            print()
+
+
+def validate_url_selectors(
+    url: str, rules: dict[str, dict], use_chrome: bool = False
+) -> dict[str, dict]:
+    """验证 URL 的 CSS 选择器
+
+    Args:
+        url: 目标 URL
+        rules: 规则字典
+        use_chrome: 是否使用 Chrome 渲染
+
+    Returns:
+        验证结果
+    """
+    import subprocess
+
+    # 获取 HTML
+    cmd = ["uv", "run", "html-reader-llm", "fetch", url]
+    if use_chrome:
+        cmd.append("--chrome")
+
+    result = subprocess.run(  # noqa: S603
+        cmd, capture_output=True, text=True, encoding="utf-8"
+    )
+
+    if result.returncode != 0:
+        raise RuntimeError(f"获取页面失败: {result.stderr}")
+
+    html = result.stdout
+
+    # 验证选择器
+    validator = SelectorValidator(html)
+    validation_results = validator.validate_rules(rules)
+    validator.print_validation_report(validation_results)
+
+    return validation_results

html_reader_llm/settings.py

@@ -0,0 +1,73 @@
+"""Environment-variable-driven configuration for html-reader-llm.
+
+All constants read from HTMLREADER_* env vars with sensible defaults.
+Loads .env file via python-dotenv if present (does not override existing env).
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Load .env from project root (if exists), does not override existing env vars
+load_dotenv(Path(__file__).resolve().parent.parent.parent / ".env")
+
+
+def _env_int(name: str, default: int) -> int:
+    """Read an integer from HTMLREADER_<name>, fallback to default."""
+    val = os.environ.get(f"HTMLREADER_{name}")
+    if val is None:
+        return default
+    try:
+        return int(val)
+    except ValueError:
+        return default
+
+
+def _env_list(name: str, default: list[str]) -> list[str]:
+    """Read a comma-separated list from HTMLREADER_<name>, fallback to default."""
+    val = os.environ.get(f"HTMLREADER_{name}")
+    if not val:
+        return default
+    return [item.strip() for item in val.split(",") if item.strip()]
+
+
+# --- Text truncation (units: Chinese chars / English words) ---
+TEXT_HEAD = _env_int("TEXT_HEAD", 10)
+TEXT_TAIL = _env_int("TEXT_TAIL", 5)
+
+# --- Script inline code truncation ---
+SCRIPT_HEAD = _env_int("SCRIPT_HEAD", 5)
+SCRIPT_TAIL = _env_int("SCRIPT_TAIL", 3)
+
+# --- Image alt truncation (chars) ---
+MEDIA_ALT_HEAD = _env_int("MEDIA_ALT_HEAD", 10)
+
+# --- Attribute mode: "blacklist" or "whitelist" ---
+ATTR_MODE = os.environ.get("HTMLREADER_ATTR_MODE", "blacklist")
+
+# --- Attribute lists (comma-separated) ---
+KEEP_ATTRS: list[str] = _env_list(
+    "KEEP_ATTRS", ["id", "class", "href", "src", "alt", "title", "role"]
+)
+REMOVE_ATTRS: list[str] = _env_list("REMOVE_ATTRS", ["style"])
+
+# --- CSS selectors for node removal / retention ---
+REMOVE_SELECTORS: list[str] = _env_list("REMOVE_SELECTORS", [])
+KEEP_SELECTORS: list[str] = _env_list("KEEP_SELECTORS", [])
+
+# --- data-* attributes: always keep (0=off, 1=on) ---
+KEEP_DATA_ATTRS = bool(_env_int("KEEP_DATA_ATTRS", 1))
+
+# --- List truncation threshold ---
+LIST_TRUNCATE_THRESHOLD = _env_int("LIST_TRUNCATE_THRESHOLD", 5)
+LIST_TRUNCATE_KEEP_HEAD = _env_int("LIST_TRUNCATE_KEEP_HEAD", 3)
+LIST_TRUNCATE_KEEP_TAIL = _env_int("LIST_TRUNCATE_KEEP_TAIL", 1)
+
+# --- LLM configuration ---
+LLM_BASE_URL = os.environ.get("HTMLREADER_LLM_BASE_URL", "")
+LLM_API_KEY = os.environ.get("HTMLREADER_LLM_API_KEY", "")
+LLM_MODEL = os.environ.get("HTMLREADER_LLM_MODEL", "")
+LLM_MAX_TOKENS = _env_int("LLM_MAX_TOKENS", 512000)