html-reader-llm 0.0.1__py3-none-any.whl

html_reader_llm/llm.py

@@ -0,0 +1,502 @@
+"""LLM analysis — page classification, metadata extraction, CSS selector rule generation.
+
+Uses OpenAI-compatible API via urllib3. trafilatura provides metadata context.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, TypedDict
+
+import urllib3
+from selectolax.lexbor import LexborHTMLParser
+
+from html_reader_llm import settings
+from html_reader_llm.render_detect import _fetch_http, detect_render
+from html_reader_llm.simplify import simplify_html
+
+logger = logging.getLogger(__name__)
+
+# =========================================================================
+# Result types
+# =========================================================================
+
+
+class SelectorRule(TypedDict):
+    css: str
+    method: str  # "$text", "$html", "@attr_name"
+
+
+class ListRule(TypedDict):
+    name: str
+    is_main: bool
+    selectors: dict[str, SelectorRule]
+
+
+class DetailRules(TypedDict, total=False):
+    title: SelectorRule | None
+    author: SelectorRule | None
+    date: SelectorRule | None
+    description: SelectorRule | None
+    image: SelectorRule | None
+    categories: SelectorRule | None
+    tags: SelectorRule | None
+    content: SelectorRule | None
+
+
+class AnalysisResult(TypedDict):
+    page_type: str
+    metadata: dict[str, Any]
+    list_rules: list[ListRule] | None
+    detail_rules: DetailRules | None
+    raw_llm_response: str
+
+
+# =========================================================================
+# Token estimation
+# =========================================================================
+
+
+def _estimate_tokens(text: str) -> int:
+    """Rough token count estimation (len/3 for CJK/English mix)."""
+    return len(text) // 3
+
+
+# =========================================================================
+# Prompt trimming
+# =========================================================================
+
+
+def _trim_to_budget(html: str, max_tokens: int) -> str:
+    """Trim HTML to fit within 50% of max_tokens.
+
+    Keeps head and tail, truncates from middle.
+    """
+    budget = max_tokens // 2
+    estimated = _estimate_tokens(html)
+    if estimated <= budget:
+        return html
+
+    # Keep 60% head, 40% tail
+    head_ratio = 0.6
+    head_chars = int(len(html) * budget / estimated * head_ratio)
+    tail_chars = int(len(html) * budget / estimated * (1 - head_ratio))
+
+    head = html[:head_chars]
+    tail = html[-tail_chars:] if tail_chars > 0 else ""
+    logger.debug(
+        "Prompt trimmed: %d -> %d tokens (budget=%d)",
+        estimated,
+        _estimate_tokens(head + tail),
+        budget,
+    )
+    return head + "\n[...truncated...]\n" + tail
+
+
+# =========================================================================
+# trafilatura metadata
+# =========================================================================
+
+
+def _extract_metadata(html: str) -> dict[str, Any]:
+    """Extract metadata from raw HTML using trafilatura."""
+    try:
+        from trafilatura.metadata import extract_metadata as _extract
+
+        meta = _extract(html)
+        result: dict[str, Any] = {}
+        for field in (
+            "title",
+            "author",
+            "date",
+            "url",
+            "sitename",
+            "description",
+            "image",
+            "categories",
+            "tags",
+        ):
+            value = getattr(meta, field, None)
+            if value:
+                result[field] = value
+        logger.debug("trafilatura metadata: %d fields extracted", len(result))
+        return result
+    except Exception as e:  # noqa: BLE001
+        logger.warning("trafilatura failed: %s", e)
+        return {}
+
+
+# =========================================================================
+# LLM API call
+# =========================================================================
+
+_SYSTEM_PROMPT = """Analyze HTML and return JSON. Two fields: page_type + rules.
+
+## Rules format
+
+Each selector: {"css": "<selector>", "method": "<method>"}
+
+Methods:
+- `$text` — inner text (for h1, p, span, a, div, td, etc.)
+- `$html` — inner HTML (for article body, content divs)
+- `@attr` — attribute value (for meta tags, links, images, time, etc.)
+
+**Standard CSS selectors ONLY.**
+FORBIDDEN: :contains(), :has(), :first, :last, :eq(), :nth(), :not(), :gt(), :lt()
+VALID: tag, #id, .class, [attr=value], tag.subclass, parent > child, a[href], meta[name="x"]
+
+## page_type: "list" or "detail"
+
+### List page output
+```json
+{"page_type":"list","list_rules":[
+  {"name":"main","is_main":true,"selectors":{
+    "title":{"css":".item-title","method":"$text"},
+    "link":{"css":".item-title","method":"@href"},
+    "pubtime":{"css":".item-date","method":"$text"}
+  }},
+  {"name":"sidebar","is_main":false,"selectors":{
+    "title":{"css":".side-title","method":"$text"},
+    "link":{"css":".side-title","method":"@href"}
+  }}
+]}
+```
+link is REQUIRED in every list rule.
+
+### Detail page output
+```json
+{"page_type":"detail","detail_rules":{
+  "title":{"css":"h1","method":"$text"},
+  "author":{"css":".author","method":"$text"},
+  "date":{"css":"meta[property='article:published_time']","method":"@content"},
+  "description":{"css":"meta[name='description']","method":"@content"},
+  "image":{"css":"meta[property='og:image']","method":"@content"},
+  "categories":{"css":".category","method":"$text"},
+  "tags":{"css":".tag","method":"$text"},
+  "content":{"css":"article .body","method":"$text"}
+}}
+```
+Omit fields that don't exist on the page. Omit empty rules.
+
+### More examples
+
+Detail with time element:
+```json
+{"title":{"css":"h1.headline","method":"$text"},"date":{"css":"time","method":"@datetime"},"content":{"css":"article","method":"$text"}}
+```
+
+List with data attributes:
+```json
+{"link":{"css":"a.article-link","method":"@href"},"title":{"css":"a.article-link","method":"$text"},"pubtime":{"css":"span.date","method":"$text"}}
+```
+
+Return ONLY valid JSON. No markdown fences. No explanation."""
+
+
+def _call_llm(
+    system_prompt: str,
+    user_prompt: str,
+    base_url: str,
+    api_key: str,
+    model: str,
+    max_completion_tokens: int = 8192,
+    temperature: float = 0.1,
+) -> str:
+    """Call OpenAI-compatible chat completions API.
+
+    Args:
+        max_completion_tokens: Max tokens for LLM completion (not prompt).
+        temperature: Sampling temperature.
+    """
+    url = f"{base_url.rstrip('/')}/chat/completions"
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        "max_tokens": max_completion_tokens,
+        "temperature": temperature,
+        "response_format": {"type": "json_object"},
+    }
+
+    logger.debug("LLM request: model=%s, prompt=%d chars", model, len(user_prompt))
+
+    http = urllib3.PoolManager()
+    resp = http.request(
+        "POST",
+        url,
+        body=json.dumps(payload).encode("utf-8"),
+        headers=headers,
+        timeout=120,
+    )
+    if resp.status != 200:
+        raise urllib3.exceptions.HTTPError(f"HTTP {resp.status}")
+    data = json.loads(resp.data.decode("utf-8"))
+
+    usage = data.get("usage", {})
+    logger.debug(
+        "LLM response: prompt_tokens=%s, completion_tokens=%s, total=%s",
+        usage.get("prompt_tokens", "?"),
+        usage.get("completion_tokens", "?"),
+        usage.get("total_tokens", "?"),
+    )
+
+    content = data["choices"][0]["message"]["content"]
+    return content
+
+
+# =========================================================================
+# Response parsing
+# =========================================================================
+
+
+def _parse_llm_response(raw: str) -> dict[str, Any]:
+    """Parse LLM JSON response with fallback."""
+    # Try direct parse
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError:
+        pass
+
+    # Try extracting JSON from markdown fences
+    match = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", raw, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except json.JSONDecodeError:
+            pass
+
+    logger.warning("Failed to parse LLM response as JSON")
+    return {}
+
+
+# =========================================================================
+# Selector validation
+# =========================================================================
+
+
+def _validate_selector(parser: Any, selector: str | dict) -> bool:
+    """Check if a CSS selector matches at least one node with extractable value.
+
+    Supports two formats:
+    - str (legacy): plain CSS selector, checks text + content attr
+    - dict (new): {"css": "...", "method": "$text|@attr"}
+    """
+    if isinstance(selector, dict):
+        css = selector.get("css", "")
+        method = selector.get("method", "$text")
+    else:
+        css = selector
+        method = "$text"
+
+    if not css:
+        return False
+
+    try:
+        nodes = parser.css(css)
+    except Exception:  # noqa: BLE001
+        return False
+    if not nodes:
+        return False
+
+    for node in nodes:
+        if method == "$text":
+            text = node.text(deep=True).strip()
+            if text:
+                return True
+        elif method == "$html":
+            html = node.html
+            if html and html.strip():
+                return True
+        elif method.startswith("@"):
+            attr_name = method[1:]
+            attr_val = (node.attributes.get(attr_name) or "").strip()
+            if attr_val:
+                return True
+        else:
+            # Unknown method, try text as fallback
+            text = node.text(deep=True).strip()
+            if text:
+                return True
+    return False
+
+
+def _validate_rules(parsed: dict[str, Any], html: str) -> dict[str, Any]:
+    """Validate LLM-generated selectors against actual HTML.
+
+    Removes selectors that don't match or produce empty values.
+    For list rules: removes rules missing required 'link' selector.
+    """
+    parser = LexborHTMLParser(html)
+    validated: dict[str, Any] = dict(parsed)
+
+    # Validate list rules
+    list_rules = parsed.get("list_rules")
+    if list_rules:
+        valid_rules = []
+        for rule in list_rules:
+            selectors = rule.get("selectors", {})
+            # link is required
+            link_sel = selectors.get("link")
+            if not link_sel or not _validate_selector(parser, link_sel):
+                logger.warning(
+                    "List rule '%s' rejected: link selector '%s' not found or empty",
+                    rule.get("name", "?"),
+                    link_sel,
+                )
+                continue
+            # Validate other selectors, remove invalid ones
+            clean_selectors: dict[str, Any] = {"link": link_sel}
+            for field in ("title", "pubtime"):
+                sel = selectors.get(field)
+                if sel and _validate_selector(parser, sel):
+                    clean_selectors[field] = sel
+                elif sel:
+                    logger.warning(
+                        "List rule '%s': %s selector '%s' removed (not found)",
+                        rule.get("name", "?"),
+                        field,
+                        sel,
+                    )
+            rule["selectors"] = clean_selectors
+            valid_rules.append(rule)
+        validated["list_rules"] = valid_rules
+
+    # Validate detail rules
+    detail_rules = parsed.get("detail_rules")
+    if detail_rules:
+        clean_detail: dict[str, Any] = {}
+        for field, sel in detail_rules.items():
+            if sel and _validate_selector(parser, sel):
+                clean_detail[field] = sel
+            elif sel:
+                logger.warning(
+                    "Detail rule: %s selector '%s' removed (not found)",
+                    field,
+                    sel,
+                )
+        validated["detail_rules"] = clean_detail if clean_detail else None
+
+    return validated
+
+
+# =========================================================================
+# Main entry point
+# =========================================================================
+
+
+def analyze_page(
+    url: str,
+    base_url: str | None = None,
+    api_key: str | None = None,
+    model: str | None = None,
+    max_tokens: int | None = None,
+    use_cache: bool = True,
+) -> AnalysisResult:
+    """Analyze a URL: classify page type and generate CSS selector rules.
+
+    Pipeline: fetch HTML → simplify → trafilatura metadata → LLM analysis.
+
+    Args:
+        url: URL to analyze.
+        base_url: LLM API base URL. Falls back to HTMLREADER_LLM_BASE_URL.
+        api_key: LLM API key. Falls back to HTMLREADER_LLM_API_KEY.
+        model: LLM model name. Falls back to HTMLREADER_LLM_MODEL.
+        max_tokens: Model context window. Falls back to HTMLREADER_LLM_MAX_TOKENS.
+        use_cache: Whether to use HTTP response cache.
+
+    Returns:
+        AnalysisResult with page_type, metadata, rules, and raw response.
+    """
+    # Resolve config
+    _base_url = base_url or settings.LLM_BASE_URL
+    _api_key = api_key or settings.LLM_API_KEY
+    _model = model or settings.LLM_MODEL
+    _max_tokens = max_tokens or settings.LLM_MAX_TOKENS
+
+    if not _base_url or not _api_key:
+        raise ValueError(
+            "LLM config required: set HTMLREADER_LLM_BASE_URL + HTMLREADER_LLM_API_KEY "
+            "or pass base_url + api_key parameters"
+        )
+
+    logger.debug("=== analyze_page: %s ===", url)
+
+    # 1. Fetch raw HTML
+    status, raw_html = _fetch_http(url, use_cache=use_cache)
+    if not raw_html:
+        raise ValueError(f"Failed to fetch URL: {url} (status={status})")
+
+    # 1.5 Detect render need
+    render_result = detect_render(url, use_cache=use_cache)
+    use_browser = render_result["use_browser"]
+    chrome_html = render_result["chrome_html"]
+    logger.debug(
+        "Render detection: use_browser=%s (dom=%.3f, text=%.3f)",
+        use_browser,
+        render_result["dom_ratio"],
+        render_result["text_similarity"],
+    )
+
+    # 2. Extract metadata (from raw HTML, before simplification)
+    metadata = _extract_metadata(raw_html)
+
+    # 3. Choose HTML for LLM: Chrome rendered if use_browser, otherwise HTTP
+    llm_source_html = chrome_html if (use_browser and chrome_html) else raw_html
+    simplified = simplify_html(llm_source_html)["html"]
+    logger.debug(
+        "Simplified: %d -> %d chars (source=%s)",
+        len(llm_source_html),
+        len(simplified),
+        "chrome" if llm_source_html is chrome_html else "http",
+    )
+
+    # 4. Trim to budget
+    trimmed = _trim_to_budget(simplified, _max_tokens)
+
+    # 5. Build prompt
+    meta_str = (
+        json.dumps(metadata, ensure_ascii=False, indent=2) if metadata else "None"
+    )
+    user_prompt = f"Metadata (from trafilatura):\n{meta_str}\n\nHTML:\n{trimmed}"
+
+    # 6. Call LLM
+    raw_response = _call_llm(
+        system_prompt=_SYSTEM_PROMPT,
+        user_prompt=user_prompt,
+        base_url=_base_url,
+        api_key=_api_key,
+        model=_model,
+    )
+
+    # 7. Parse response
+    parsed = _parse_llm_response(raw_response)
+
+    # 7.5 Validate selectors against the same HTML used for LLM analysis
+    validate_html = llm_source_html
+    parsed = _validate_rules(parsed, validate_html)
+
+    # 8. Build result
+    page_type = parsed.get("page_type", "unknown")
+    result: AnalysisResult = {
+        "page_type": page_type,
+        "metadata": metadata,
+        "list_rules": parsed.get("list_rules") if page_type == "list" else None,
+        "detail_rules": parsed.get("detail_rules") if page_type == "detail" else None,
+        "raw_llm_response": raw_response,
+    }
+
+    logger.debug(
+        "=== analyze_page done: type=%s, metadata=%d fields ===",
+        page_type,
+        len(metadata),
+    )
+
+    return result