webreader 2.3.8__py3-none-any.whl

webreader/core.py

@@ -0,0 +1,501 @@
+"""Tree-based HTML parsing engine built on top of the parser package."""
+
+import codecs
+import hashlib
+import os
+import re
+from html import escape as _escape
+from html.parser import HTMLParser as _BaseHTMLParser
+from typing import Any, Dict, Iterator, List, Optional, Union
+
+from pafer.cache import LRUCache
+from pafer.config import (
+    Config,
+    PathLike,
+    load_config as _load_config,
+    save_config as _save_config,
+)
+from pafer.exceptions import ConfigLoadError, HTMLPaferError
+from pafer.utils import extract_doctype, minify_html
+
+VOID_ELEMENTS = {
+    'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
+    'link', 'meta', 'param', 'source', 'track', 'wbr',
+}
+
+BLOCK_TAGS = {
+    'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd',
+    'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form',
+    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li',
+    'main', 'nav', 'ol', 'p', 'pre', 'section', 'table', 'tbody', 'thead',
+    'tfoot', 'tr', 'ul',
+}
+
+ROOT_TAG = ':root'
+COMMENT_TAG = ':comment'
+
+PRESERVE_WHITESPACE_TAGS = {'pre', 'textarea'}
+
+
+def detect_encoding(data: bytes, default: str = 'utf-8') -> str:
+    """Detect the encoding of raw HTML: BOM sniff, then <meta charset>."""
+    if data.startswith(codecs.BOM_UTF8):
+        return 'utf-8-sig'
+    if data.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+        return 'utf-32'
+    if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+        return 'utf-16'
+    head = data[:2048].decode('latin-1', errors='ignore')
+    match = re.search(
+        r'<meta[^>]+charset\s*=\s*["\']?\s*([\w.:+-]+)', head, re.IGNORECASE
+    )
+    if match:
+        try:
+            codecs.lookup(match.group(1))
+            return match.group(1)
+        except LookupError:
+            pass
+    return default
+
+
+class Node:
+    """A single element or comment node in the parsed HTML tree."""
+
+    def __init__(self, tag: str, attrs: Optional[Dict[str, str]] = None,
+                 parent: Optional['Node'] = None) -> None:
+        self.tag = tag
+        self.attrs: Dict[str, str] = dict(attrs or {})
+        self.children: List[Any] = []
+        self.parent: Optional[Node] = parent
+
+    @property
+    def is_comment(self) -> bool:
+        return self.tag == COMMENT_TAG
+
+    def get_attr(self, name: str, default: Optional[str] = None) -> Optional[str]:
+        return self.attrs.get(name, default)
+
+    def has_class(self, name: str) -> bool:
+        return name in (self.get_attr('class') or '').split()
+
+    def set_attr(self, name: str, value: str) -> 'Node':
+        self.attrs[name] = value
+        return self
+
+    def remove_attr(self, name: str) -> 'Node':
+        self.attrs.pop(name, None)
+        return self
+
+    def append_child(self, node: 'Node') -> 'Node':
+        if node.parent is not None:
+            node.parent.children.remove(node)
+        self.children.append(node)
+        node.parent = self
+        return self
+
+    def remove(self) -> 'Node':
+        if self.parent is not None:
+            self.parent.children.remove(self)
+            self.parent = None
+        return self
+
+    def replace_with(self, *nodes: 'Node') -> 'Node':
+        if self.parent is None:
+            raise PaferError('cannot replace a detached node')
+        siblings = self.parent.children
+        index = siblings.index(self)
+        replacements = list(nodes)
+        for node in replacements:
+            if node.parent is not None:
+                node.parent.children.remove(node)
+        siblings[index:index + 1] = replacements
+        for node in replacements:
+            node.parent = self.parent
+        self.parent = None
+        return self
+
+    def unwrap(self) -> 'Node':
+        if self.parent is None:
+            raise PaferError('cannot unwrap a detached node')
+        siblings = self.parent.children
+        index = siblings.index(self)
+        children = list(self.children)
+        siblings[index:index + 1] = children
+        for child in children:
+            if isinstance(child, Node):
+                child.parent = self.parent
+        self.children = []
+        self.parent = None
+        return self
+
+    def child_elements(self) -> List['Node']:
+        return [
+            child
+            for child in self.children
+            if isinstance(child, Node) and not child.is_comment
+        ]
+
+    def text_content(self) -> str:
+        parts: List[str] = []
+        for child in self.children:
+            if isinstance(child, str):
+                parts.append(child)
+            else:
+                parts.append(child.text_content())
+        return ''.join(parts)
+
+    def iter_elements(self) -> Iterator['Node']:
+        for child in self.children:
+            if isinstance(child, Node) and not child.is_comment:
+                yield child
+                yield from child.iter_elements()
+
+    def to_html(self) -> str:
+        if self.is_comment:
+            return f'<!--{self.text_content()}-->'
+        attrs = ''.join(
+            f' {name}' if value == '' else f' {name}="{_escape(value, quote=True)}"'
+            for name, value in self.attrs.items()
+        )
+        if self.tag in VOID_ELEMENTS:
+            return f'<{self.tag}{attrs}>'
+        inner = ''.join(
+            _escape(child, quote=False) if isinstance(child, str) else child.to_html()
+            for child in self.children
+        )
+        return f'<{self.tag}{attrs}>{inner}</{self.tag}>'
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            'tag': self.tag,
+            'attrs': dict(self.attrs),
+            'text': self.text_content().strip(),
+            'html': self.to_html(),
+        }
+
+    def __repr__(self) -> str:
+        return f'<Node {self.tag}>'
+
+
+class _TreeBuilder(_BaseHTMLParser):
+    """Builds a Node tree from HTML using the stdlib tokenizer."""
+
+    def __init__(self, config: Config) -> None:
+        super().__init__(convert_charrefs=True)
+        self.config = config
+        self.root = Node(ROOT_TAG)
+        self._stack: List[Node] = [self.root]
+        self.comments: List[str] = []
+        self.errors: List[str] = []
+        self._max_depth = int(config.get('max_depth', 100) or 100)
+        self._preserve_comments = bool(config.get('preserve_comments', False))
+        self._strip_whitespace = bool(config.get('strip_whitespace', True))
+        self._strict = bool(config.get('strict_mode', False))
+
+    def _current(self) -> Node:
+        return self._stack[-1]
+
+    def _attach(self, node: Node) -> None:
+        parent = self._current()
+        if len(self._stack) <= self._max_depth:
+            parent.children.append(node)
+            node.parent = parent
+        elif self._strict:
+            self.errors.append(
+                f'<{node.tag}> exceeds max_depth={self._max_depth}'
+            )
+        self._stack.append(node)
+
+    def handle_starttag(self, tag, attrs) -> None:
+        node = Node(tag, {k: (v if v is not None else '') for k, v in attrs})
+        self._attach(node)
+        if tag in VOID_ELEMENTS:
+            self._stack.pop()
+
+    def handle_startendtag(self, tag, attrs) -> None:
+        node = Node(tag, {k: (v if v is not None else '') for k, v in attrs})
+        if len(self._stack) <= self._max_depth:
+            parent = self._current()
+            parent.children.append(node)
+            node.parent = parent
+
+    def handle_endtag(self, tag) -> None:
+        if tag in VOID_ELEMENTS:
+            return
+        for i in range(len(self._stack) - 1, 0, -1):
+            if self._stack[i].tag == tag:
+                if len(self._stack) - 1 > i:
+                    unclosed = [n.tag for n in self._stack[i + 1:]]
+                    self.errors.append(
+                        f'unclosed tag(s) <{"> <".join(unclosed)}> before </{tag}>'
+                    )
+                del self._stack[i:]
+                return
+        self.errors.append(f'closing tag </{tag}> has no matching open tag')
+
+    def handle_data(self, data) -> None:
+        if not data:
+            return
+        if self._strip_whitespace and not any(
+            node.tag in PRESERVE_WHITESPACE_TAGS for node in self._stack
+        ):
+            data = re.sub(r'\s+', ' ', data)
+        self._current().children.append(data)
+
+    def handle_comment(self, data) -> None:
+        self.comments.append(data)
+        if self._preserve_comments:
+            comment = Node(COMMENT_TAG)
+            comment.children.append(data)
+            if len(self._stack) <= self._max_depth:
+                parent = self._current()
+                parent.children.append(comment)
+                comment.parent = parent
+
+    def close(self) -> None:
+        super().close()
+        if len(self._stack) > 1:
+            unclosed = [n.tag for n in self._stack[1:]]
+            self.errors.append(
+                f'unclosed tag(s) at end of input: <{"> <".join(unclosed)}>'
+            )
+
+
+class ParsedDocument:
+    """Structured, lazily-enriched result of parsing an HTML document."""
+
+    def __init__(self, doctype: Optional[str], root: Node,
+                 comments: Optional[List[str]] = None,
+                 config: Optional[Config] = None,
+                 errors: Optional[List[str]] = None) -> None:
+        self.doctype = doctype
+        self.root = root
+        self.comments = comments or []
+        self.errors = errors or []
+        self.config = config or Config()
+        self._metadata: Optional[Dict[str, Any]] = None
+        self._links: Optional[List[Dict[str, Any]]] = None
+        self._tables: Optional[List[Dict[str, Any]]] = None
+        self._text: Optional[str] = None
+        self._images: Optional[List[Dict[str, Any]]] = None
+        self._forms: Optional[List[Dict[str, Any]]] = None
+        self._headings: Optional[Dict[str, Any]] = None
+        self._scripts: Optional[List[Dict[str, Any]]] = None
+        self._stylesheets: Optional[List[Dict[str, Any]]] = None
+        self._markdown: Optional[str] = None
+
+    def metadata(self) -> Dict[str, Any]:
+        from .extractors import extract_metadata
+        if self._metadata is None:
+            self._metadata = extract_metadata(self)
+        return self._metadata
+
+    def links(self, base_url: Optional[str] = None) -> List[Dict[str, Any]]:
+        from .extractors import extract_links
+        if base_url is None and self._links is not None:
+            return self._links
+        result = extract_links(self, base_url=base_url)
+        if base_url is None:
+            self._links = result
+        return result
+
+    def tables(self) -> List[Dict[str, Any]]:
+        from .extractors import extract_tables
+        if self._tables is None:
+            self._tables = extract_tables(self)
+        return self._tables
+
+    def text(self) -> str:
+        from .extractors import extract_text
+        if self._text is None:
+            self._text = extract_text(self)
+        return self._text
+
+    def find_all(self, selector: str) -> List[Node]:
+        from .selectors import find_all
+        return find_all(self.root, selector)
+
+    def find(self, selector: str) -> Optional[Node]:
+        from .selectors import find
+        return find(self.root, selector)
+
+    def images(self) -> List[Dict[str, Any]]:
+        from .extractors import extract_images
+        if self._images is None:
+            self._images = extract_images(self)
+        return self._images
+
+    def forms(self) -> List[Dict[str, Any]]:
+        from .extractors import extract_forms
+        if self._forms is None:
+            self._forms = extract_forms(self)
+        return self._forms
+
+    def headings(self) -> Dict[str, Any]:
+        from .extractors import extract_headings
+        if self._headings is None:
+            self._headings = extract_headings(self)
+        return self._headings
+
+    def scripts(self) -> List[Dict[str, Any]]:
+        from .extractors import extract_scripts
+        if self._scripts is None:
+            self._scripts = extract_scripts(self)
+        return self._scripts
+
+    def stylesheets(self) -> List[Dict[str, Any]]:
+        from .extractors import extract_stylesheets
+        if self._stylesheets is None:
+            self._stylesheets = extract_stylesheets(self)
+        return self._stylesheets
+
+    def to_markdown(self) -> str:
+        from .markdown import document_to_markdown
+        if self._markdown is None:
+            self._markdown = document_to_markdown(self)
+        return self._markdown
+
+    def to_html(self) -> str:
+        return ''.join(
+            _escape(child, quote=False) if isinstance(child, str) else child.to_html()
+            for child in self.root.children
+        )
+
+    def stats(self) -> Dict[str, int]:
+        element_count = 0
+        max_depth = 0
+
+        def walk(node: Node, depth: int) -> None:
+            nonlocal element_count, max_depth
+            for child in node.children:
+                if isinstance(child, Node) and not child.is_comment:
+                    element_count += 1
+                    if depth + 1 > max_depth:
+                        max_depth = depth + 1
+                    walk(child, depth + 1)
+
+        walk(self.root, 0)
+        return {
+            'elements': element_count,
+            'comments': len(self.comments),
+            'max_depth': max_depth,
+        }
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            'doctype': self.doctype,
+            'stats': self.stats(),
+            'metadata': self.metadata(),
+            'links': self.links(),
+            'tables': self.tables(),
+            'text': self.text(),
+            'comments': list(self.comments),
+            'errors': list(self.errors),
+        }
+
+    def __repr__(self) -> str:
+        return f'<ParsedDocument {self.stats()["elements"]} elements>'
+
+
+class Engine:
+    """High-level HTML processing engine configured by a pafer Config."""
+
+    def __init__(self, config: Optional[Config] = None, use_cache: bool = True) -> None:
+        self.config = config or Config()
+        self.cache: Optional[LRUCache] = None
+        if use_cache:
+            self.cache = LRUCache(max_size=int(self.config.get('cache_size', 50) or 50))
+
+    def parse(self, html: Union[str, bytes]) -> ParsedDocument:
+        """Parse HTML (str or bytes) into a ParsedDocument."""
+        if isinstance(html, bytes):
+            encoding = detect_encoding(
+                html,
+                default=str(self.config.get('encoding', 'utf-8') or 'utf-8'),
+            )
+            html = html.decode(encoding, errors='replace')
+        key = hashlib.sha256(html.encode('utf-8', errors='replace')).hexdigest()
+        if self.cache is not None:
+            cached = self.cache.get(key)
+            if cached is not None:
+                return cached
+        self._check_size(html)
+        builder = _TreeBuilder(self.config)
+        try:
+            builder.feed(html)
+            builder.close()
+        except PaferError:
+            raise
+        except Exception as exc:
+            raise PaferError(f'failed to parse HTML: {exc}') from exc
+        if bool(self.config.get('strict_mode', False)) and builder.errors:
+            summary = '; '.join(builder.errors[:5])
+            raise PaferError(f'malformed HTML (strict mode): {summary}')
+        doc = ParsedDocument(
+            doctype=extract_doctype(html),
+            root=builder.root,
+            comments=builder.comments,
+            config=self.config,
+            errors=builder.errors,
+        )
+        if self.cache is not None:
+            self.cache.set(key, doc)
+        return doc
+
+    def _check_size(self, html: str) -> None:
+        cap = self.config.get('max_file_size_mb', 10)
+        if not cap:
+            return
+        size_mb = len(html.encode('utf-8', errors='replace')) / (1024 * 1024)
+        if size_mb > float(cap):
+            raise PaferError(
+                f'input of {size_mb:.2f} MB exceeds max_file_size_mb={cap}'
+            )
+
+    def minify(self, html: str) -> str:
+        """Minify HTML honoring remove_comments from the configuration."""
+        return minify_html(html, remove_comments=bool(self.config.get('remove_comments', True)))
+
+
+def parse_raw(html: Union[str, bytes],
+              config: Optional[Config] = None) -> ParsedDocument:
+    """Parse with whitespace preserved; used by pretty and markdown."""
+    cfg = (config or Config()).update({'strip_whitespace': False})
+    return Engine(cfg, use_cache=False).parse(html)
+
+def get_config():
+    """Return the active configuration object."""
+    return config
+
+def get_engine() -> Engine:
+    """Return the active parsing engine."""
+    return _engine
+
+def load_config(path: Optional[PathLike] = None) -> Dict[str, Any]:
+    """Load user config."""
+    global config, _engine
+    try:
+        cfg = _load_config(path)
+    except ConfigLoadError:
+        cfg = {}
+    try:
+        _save_config(cfg)
+    except (ConfigLoadError, OSError):
+        pass
+    config = cfg
+    _engine = Engine(cfg)
+    return cfg
+
+def save_config(cfg) -> None:
+    try:
+        _save_config(cfg)
+    except OSError:
+        pass
+
+
+try:
+    config = _load_config(None)
+except Exception:
+    config = Config()
+
+_engine = Engine(config)