html-reader-llm 0.0.1__py3-none-any.whl

html_reader_llm/simplify.py

@@ -0,0 +1,547 @@
+"""HTML simplification pipeline — transforms raw HTML into LLM-friendly format.
+
+Fixed 7-step pipeline:
+1. Strip noise tags (script/style/noscript etc.) + comments
+2. Simplify media (img/video/audio → placeholders)
+3. Clean attributes (blacklist/whitelist dual mode) + URL normalization
+4. Truncate repeated list items
+5. Truncate text nodes (regular + script inline)
+6. Unwrap bare wrapper tags
+7. Clean whitespace
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import PurePosixPath
+from typing import Any
+from urllib.parse import urlparse
+
+from selectolax.lexbor import LexborHTMLParser
+
+from html_reader_llm import settings  # noqa: E402 — relative import for package copy
+
+logger = logging.getLogger(__name__)
+
+# --- Noise tags: entire subtree removed ---
+NOISE_TAGS = frozenset(
+    {
+        "style",
+        "noscript",
+        "iframe",
+        "object",
+        "embed",
+        "base",
+        "font",
+        "link",
+        "br",
+        "hr",
+        "header",
+        "nav",
+        "footer",
+        "svg",
+        "path",
+    }
+)
+
+# --- Keep tags: never unwrapped by bare-tag removal ---
+KEEP_TAGS = frozenset(
+    {
+        "html",
+        "head",
+        "body",
+        "title",
+        "meta",
+        "table",
+        "thead",
+        "tbody",
+        "tfoot",
+        "tr",
+        "th",
+        "td",
+        "ul",
+        "ol",
+        "li",
+        "dl",
+        "dt",
+        "dd",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "p",
+        "a",
+        "img",
+    }
+)
+
+# --- Structural tags: protect parent from unwrap ---
+STRUCTURE_TAGS = frozenset(
+    {
+        "ul",
+        "ol",
+        "li",
+        "dl",
+        "dt",
+        "dd",
+        "table",
+        "thead",
+        "tbody",
+        "tfoot",
+        "tr",
+        "th",
+        "td",
+    }
+)
+
+# --- Truncatable item tags ---
+TRUNCATABLE_ITEM_TAGS = frozenset({"li", "tr"})
+
+# --- Unit regex for text truncation ---
+_UNIT_RE = re.compile(
+    r"\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:\s+\d{1,2}:\d{2}(?::\d{2})?)?"
+    r"|\d{1,2}:\d{2}(?::\d{2})?"
+    r"|[\u4e00-\u9fff\u3400-\u4dbf]"
+    r"|[a-zA-Z]+"
+    r"|\d+"
+)
+
+_MULTI_WS_RE = re.compile(r"\s+")
+_SHORTENED_PLACEHOLDER = "[省略{n}字]"
+
+# --- Default noise CSS selectors ---
+DEFAULT_NOISE_SELECTORS: tuple[str, ...] = (
+    ".header",
+    ".footer",
+    ".nav",
+    ".navbar",
+    ".sidebar",
+    "#sidebar",
+    ".ad",
+    ".ads",
+    ".advertisement",
+    "[class*='banner']",
+    "[class*='cookie']",
+    "[class*='popup']",
+    "[class*='modal']",
+    "[id*='ad-']",
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _walk_nodes(node: Any) -> Any:
+    """Iterative DFS over DOM tree, yielding all nodes including text nodes.
+
+    selectolax lexbor's traverse() skips text nodes,
+    so we manually walk via first_child / next.
+    Explicit stack avoids recursion depth issues on deep DOMs.
+    """
+    stack = [node]
+    while stack:
+        current = stack.pop()
+        yield current
+        children: list[Any] = []
+        child = current.first_child
+        while child is not None:
+            children.append(child)
+            child = child.next
+        stack.extend(reversed(children))
+
+
+def _normalize_url(url: str) -> str:
+    """Convert a URL to its last path segment, stripping query and fragment.
+
+    Examples:
+        /static/jquery.min.js          → jquery.min.js
+        https://cdn.example.com/a.js   → a.js
+        /page?id=1#section             → page
+        /                              → /
+    """
+    if not url or url.strip() == "/":
+        return url.strip() if url else ""
+    try:
+        parsed = urlparse(url)
+        path = parsed.path
+    except Exception:  # noqa: BLE001
+        path = url
+    basename = PurePosixPath(path).name
+    return basename if basename else path
+
+
+def _human_size(data_len: int) -> str:
+    """Convert byte count to human-readable size string."""
+    if data_len < 1024:
+        return f"{data_len}B"
+    if data_len < 1024 * 1024:
+        return f"{data_len / 1024:.1f}KB"
+    return f"{data_len / (1024 * 1024):.1f}MB"
+
+
+# ---------------------------------------------------------------------------
+# Step 1: Strip noise tags + comments
+# ---------------------------------------------------------------------------
+
+
+def _strip_noise(parser: LexborHTMLParser) -> None:
+    """Remove noise tag subtrees and comment nodes."""
+    to_remove = []
+    for node in _walk_nodes(parser.root):
+        if node.is_comment_node:
+            to_remove.append(node)
+            continue
+        if not node.is_element_node:
+            continue
+        if node.tag.lower() in NOISE_TAGS:
+            to_remove.append(node)
+    for node in to_remove:
+        node.remove()
+
+
+# ---------------------------------------------------------------------------
+# Step 2: Simplify media → placeholders
+# ---------------------------------------------------------------------------
+
+
+def _simplify_media(parser: LexborHTMLParser) -> None:
+    """Replace img/video/audio/source/track with text placeholders.
+
+    img → [img: <filename|size> <alt_first_10_chars>]
+    video → [video]
+    audio/source/track → [audio]
+    """
+    for node in _walk_nodes(parser.root):
+        if not node.is_element_node:
+            continue
+        tag = node.tag.lower()
+        if tag == "img":
+            attrs = node.attributes
+            alt = (attrs.get("alt") or "").strip()
+            src = attrs.get("src", "")
+            if src.startswith("data:"):
+                # base64: show human-readable size
+                # data:image/png;base64,XXXX → estimate decoded size
+                _, _, data = src.partition(",")
+                size_str = _human_size(int(len(data) * 3 / 4)) if data else "0B"
+                identity = size_str
+            else:
+                identity = _normalize_url(src) if src else "unknown"
+            alt_text = alt[: settings.MEDIA_ALT_HEAD] if alt else ""
+            if alt_text:
+                node.replace_with(f"[img: {identity} {alt_text}]")
+            else:
+                node.replace_with(f"[img: {identity}]")
+        elif tag in ("video", "audio", "source", "track"):
+            node.replace_with("[video]" if tag == "video" else "[audio]")
+
+
+# ---------------------------------------------------------------------------
+# Step 3: Clean attributes + URL normalization
+# ---------------------------------------------------------------------------
+
+
+def _is_data_attr(attr: str) -> bool:
+    """Check if an attribute is a data-* attribute."""
+    return attr.startswith("data-")
+
+
+def _attrs_to_remove(attrs: Any) -> list[str]:
+    """Compute which attribute names to delete based on mode and settings.
+
+    Blacklist mode (default): delete only REMOVE_ATTRS entries.
+    Whitelist mode: delete everything except KEEP_ATTRS + data-* (if enabled).
+    """
+    if settings.ATTR_MODE == "whitelist":
+        return [
+            k
+            for k in attrs
+            if k.lower() not in settings.KEEP_ATTRS
+            and not (_is_data_attr(k) and settings.KEEP_DATA_ATTRS)
+        ]
+    # Blacklist mode: only remove listed attrs
+    return [k for k in attrs if k.lower() in settings.REMOVE_ATTRS]
+
+
+def _clean_attributes_dom(parser: LexborHTMLParser) -> None:
+    """Clean attributes using blacklist/whitelist mode.
+
+    Special handling for <a> tags:
+    - href empty → remove
+    - href="#" → unwrap if text content, else remove
+    - normal href → keep href (+ data-* attrs), remove others
+
+    For all other elements:
+    - blacklist mode: remove REMOVE_ATTRS, keep data-* if enabled
+    - whitelist mode: keep only KEEP_ATTRS + data-*
+
+    NOTE: Must use node.attrs (LexborAttributes) not node.attributes (copy dict).
+    """
+    to_remove: list[Any] = []
+    to_unwrap: list[Any] = []
+    for node in _walk_nodes(parser.root):
+        if not node.is_element_node:
+            continue
+        tag = node.tag.lower()
+        # Use node.attrs (LexborAttributes) for in-place DOM modification
+        node_attrs = node.attrs
+        attrs_dict = node.attributes  # copy, for reading values
+
+        if tag == "a":
+            href = attrs_dict.get("href", "")
+            if not href:
+                to_remove.append(node)
+                continue
+            if href.strip() == "#":
+                has_text = False
+                child = node.first_child
+                while child is not None:
+                    if child.is_text_node:
+                        t = child.text_lexbor()
+                        if t and t.strip():
+                            has_text = True
+                            break
+                    child = child.next
+                if has_text:
+                    to_unwrap.append(node)
+                else:
+                    to_remove.append(node)
+                continue
+            # Normal anchor: keep href + data-*, remove others
+            keys_to_del = [
+                k
+                for k in attrs_dict
+                if k.lower() != "href"
+                and not (_is_data_attr(k) and settings.KEEP_DATA_ATTRS)
+            ]
+            for k in keys_to_del:
+                del node_attrs[k]
+            # Normalize href via attrs proxy
+            node_attrs["href"] = _normalize_url(href)
+        elif tag == "script":
+            # Script: keep src + type + data-*, normalize src
+            keys_to_del = [
+                k
+                for k in attrs_dict
+                if k.lower() not in ("src", "type")
+                and not (_is_data_attr(k) and settings.KEEP_DATA_ATTRS)
+            ]
+            for k in keys_to_del:
+                del node_attrs[k]
+            if "src" in attrs_dict:
+                node_attrs["src"] = _normalize_url(attrs_dict["src"])
+        else:
+            # General elements: apply attribute mode
+            keys_to_del = _attrs_to_remove(attrs_dict)
+            for k in keys_to_del:
+                del node_attrs[k]
+    for node in to_remove:
+        node.remove()
+    for node in to_unwrap:
+        node.unwrap()
+
+
+# ---------------------------------------------------------------------------
+# Step 6: Unwrap bare wrapper tags
+# ---------------------------------------------------------------------------
+
+_MAX_UNWRAP_ROUNDS = 100
+
+
+def _is_bare(node: Any) -> bool:
+    """Check if node has no meaningful attributes after filtering."""
+    if not node.attributes:
+        return True
+    # Has attributes only if something remains after attribute cleaning
+    return False
+
+
+def _has_structure_descendant(node: Any) -> bool:
+    """Check if direct children contain structural tags (ul/ol/table/tr/td etc.)."""
+    child = node.first_child
+    while child is not None:
+        if not child.is_text_node and child.tag.lower() in STRUCTURE_TAGS:
+            return True
+        child = child.next
+    return False
+
+
+def _unwrap_bare_tags(parser: LexborHTMLParser) -> None:
+    """Remove bare wrapper tags (no attributes), keeping children.
+
+    Protection rules:
+    - KEEP_TAGS never unwrapped
+    - Parent of structural tags never unwrapped
+    """
+    for _ in range(_MAX_UNWRAP_ROUNDS):
+        to_unwrap: list[Any] = []
+        for node in _walk_nodes(parser.root):
+            if not node.is_element_node:
+                continue
+            tag = node.tag.lower()
+            if tag in KEEP_TAGS:
+                continue
+            if not _is_bare(node):
+                continue
+            if _has_structure_descendant(node):
+                continue
+            to_unwrap.append(node)
+        if not to_unwrap:
+            break
+        # Process leaf-upward to avoid parent-child conflicts
+        for node in reversed(to_unwrap):
+            if node.first_child is None:
+                node.remove()
+            else:
+                node.unwrap()
+
+
+# ---------------------------------------------------------------------------
+# Step 5: Truncate text nodes (regular + script inline)
+# ---------------------------------------------------------------------------
+
+
+def shorten_text(text: str, head: int, tail: int) -> str:
+    """Truncate text by keeping head and tail units, inserting a placeholder.
+
+    Units: Chinese chars, English words, date/time patterns, numbers.
+    """
+    if head <= 0 and tail <= 0:
+        return text
+    matches = list(_UNIT_RE.finditer(text))
+    total = len(matches)
+    if total <= head + tail:
+        return text
+    head_end = matches[head - 1].end() if head > 0 else 0
+    tail_start = matches[-tail].start() if tail > 0 else len(text)
+    hidden = total - head - tail
+    if hidden <= 6:
+        return text
+    return text[:head_end] + _SHORTENED_PLACEHOLDER.format(n=hidden) + text[tail_start:]
+
+
+def _truncate_text_nodes(parser: LexborHTMLParser) -> None:
+    """Truncate all text nodes and script inline code."""
+    ops: list[tuple[Any, str | None]] = []
+    for node in _walk_nodes(parser.root):
+        if not node.is_text_node:
+            continue
+        parent = node.parent
+        if parent is None or parent.is_text_node:
+            continue
+        parent_tag = parent.tag.lower() if parent.is_element_node else ""
+        # Skip title tag text
+        if parent_tag == "title":
+            continue
+        original = node.text_lexbor()
+        if not original.strip():
+            ops.append((node, None))
+            continue
+        cleaned = _MULTI_WS_RE.sub(" ", original).strip()
+        # Script inline code uses SCRIPT_HEAD/TAIL, others use TEXT_HEAD/TAIL
+        if parent_tag == "script":
+            shortened = shorten_text(
+                cleaned, settings.SCRIPT_HEAD, settings.SCRIPT_TAIL
+            )
+        else:
+            shortened = shorten_text(cleaned, settings.TEXT_HEAD, settings.TEXT_TAIL)
+        if shortened != original:
+            ops.append((node, shortened))
+    for node, new_text in ops:
+        if new_text is None:
+            node.remove()
+        else:
+            node.replace_with(new_text)
+
+
+# ---------------------------------------------------------------------------
+# Step 4: Truncate repeated list items
+# ---------------------------------------------------------------------------
+
+
+def _truncate_repeated_items(parser: LexborHTMLParser) -> None:
+    """Truncate consecutive li/tr items exceeding threshold, keep head+tail, insert marker."""
+    containers: list[list[Any]] = []
+    for node in _walk_nodes(parser.root):
+        if not node.is_element_node:
+            continue
+        if node.tag.lower() not in ("ul", "ol", "table", "tbody", "thead", "tfoot"):
+            continue
+        items: list[Any] = []
+        child = node.first_child
+        while child is not None:
+            if child.is_element_node and child.tag.lower() in TRUNCATABLE_ITEM_TAGS:
+                items.append(child)
+            child = child.next
+        threshold = (
+            settings.LIST_TRUNCATE_KEEP_HEAD + settings.LIST_TRUNCATE_KEEP_TAIL + 1
+        )
+        if len(items) > threshold:
+            containers.append(items)
+    for items in containers:
+        total = len(items)
+        keep_head = settings.LIST_TRUNCATE_KEEP_HEAD
+        keep_tail = settings.LIST_TRUNCATE_KEEP_TAIL
+        to_remove = items[keep_head:-keep_tail] if keep_tail > 0 else items[keep_head:]
+        omitted = len(to_remove)
+        # Insert marker before first tail item
+        anchor = items[-keep_tail] if keep_tail > 0 else items[-1]
+        anchor.insert_before(f"[共{total}条，省略{omitted}条]")
+        for item in to_remove:
+            item.remove()
+
+
+# ---------------------------------------------------------------------------
+# Step 7: Whitespace normalization
+# ---------------------------------------------------------------------------
+
+
+def _clean_whitespace(html: str) -> str:
+    """Collapse multi-char whitespace between and within tags."""
+    return re.sub(r">\s+<", "> <", html)
+
+
+# ---------------------------------------------------------------------------
+# Pipeline assembly
+# ---------------------------------------------------------------------------
+
+_PIPELINE_STEPS = [
+    ("strip_noise", _strip_noise),
+    ("simplify_media", _simplify_media),
+    ("clean_attributes", _clean_attributes_dom),
+    ("truncate_lists", _truncate_repeated_items),
+    ("truncate_text", _truncate_text_nodes),
+    ("unwrap_bare_tags", _unwrap_bare_tags),
+]
+
+
+def simplify_html(html: str) -> dict[str, Any]:
+    """Run the 7-step simplification pipeline.
+
+    Returns:
+        dict with keys:
+        - "html": simplified HTML string
+        - "stats": dict[str, dict[str, int]] mapping step name to
+          {char_before, char_after}
+    """
+    stats: dict[str, dict[str, int]] = {}
+    parser = LexborHTMLParser(html)
+
+    for step_name, step_fn in _PIPELINE_STEPS:
+        char_before = len(parser.html)
+        step_fn(parser)
+        char_after = len(parser.html)
+        stats[step_name] = {"char_before": char_before, "char_after": char_after}
+
+    # Final whitespace cleanup (string-level)
+    ws_before = len(parser.html)
+    result_html = _clean_whitespace(parser.html)
+    ws_after = len(result_html)
+    stats["clean_whitespace"] = {"char_before": ws_before, "char_after": ws_after}
+
+    return {"html": result_html, "stats": stats}

html_reader_llm-0.0.1.dist-info/METADATA

@@ -0,0 +1,52 @@
+Metadata-Version: 2.4
+Name: html-reader-llm
+Version: 0.0.1
+Summary: HTML simplification and intelligent extraction for LLM
+Author-email: Clericpy <clericpy@gmail.com>
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: minify-html>=0.18.1
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: selectolax>=0.4.7
+Requires-Dist: trafilatura>=2.2.0
+
+# html-reader-llm
+
+HTML simplification and intelligent extraction for LLM.
+
+## Features
+
+- HTML simplification for LLM consumption
+- Render detection (HTTP vs Chrome)
+- LLM-powered page classification and CSS selector generation
+- CSS selector validation
+
+## Installation
+
+```bash
+pip install html-reader-llm
+```
+
+## Usage
+
+```bash
+# Simplify HTML
+html-reader-llm simplify < page.html
+
+# Detect if browser rendering is needed
+html-reader-llm detect https://example.com
+
+# Full analysis with LLM
+html-reader-llm analyze https://example.com
+
+# Fetch HTML
+html-reader-llm fetch https://example.com
+
+# Validate CSS selectors
+html-reader-llm validate https://example.com --rules-file selectors.json
+```
+
+## License
+
+MIT

html_reader_llm-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+html_reader_llm/__init__.py,sha256=nvxvpTlcArqeCGCqqTa9571dAJSvJtGHEaz2md3rqd4,1210
+html_reader_llm/browser.py,sha256=Kw7d70YTyL0m6vK6slO4icYYu5iJqVYcYndmLA4_NoE,4717
+html_reader_llm/cli.py,sha256=oOQJU3wrOTZ0OQUMypchiBa03k-0A5_QuuqAHfGudA0,19957
+html_reader_llm/llm.py,sha256=vWD6ZvNBSzkjVF6SaKOQOfYVc4lozIPDKgTJTCqENOY,15824
+html_reader_llm/render_detect.py,sha256=XTPkg8jsB9py31LkJ9_H2pi1BxjCpPTVY4Q8QL3fRwE,12607
+html_reader_llm/selector_validator.py,sha256=sZsEW9Rfx_ei-xZOqmoY4miojehjk-DHfbex5U52n94,5495
+html_reader_llm/settings.py,sha256=jV3jS-u1wtgSirLDqI_DVraet12Rqgt0LoKtOzJcLtE,2525
+html_reader_llm/simplify.py,sha256=XuP3KP_JeXvvsigWo9WoGLj2do7GLLKB7F3TXNIvzy8,17345
+html_reader_llm-0.0.1.dist-info/entry_points.txt,sha256=eA-OAGTqIjfT4ATIHu5dSj6aJn6Bb3cfyUL1PR7gSEo,60
+html_reader_llm-0.0.1.dist-info/licenses/LICENSE,sha256=2wBhZrpJjfa6xenTNHJNPMLlhljJ3R6p8zhA3SGNTfo,1086
+html_reader_llm-0.0.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+html_reader_llm-0.0.1.dist-info/METADATA,sha256=1Ql68mmnTjL1ZMbm8Z6ivM6LnSqp-VMpLJfRzpRx97E,1107
+html_reader_llm-0.0.1.dist-info/RECORD,,

html_reader_llm-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py3-none-any

html_reader_llm-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+html-reader-llm=html_reader_llm.cli:main
+

html_reader_llm-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ClericPy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.