webreader 2.3.8__py3-none-any.whl

webreader/markdown.py

@@ -0,0 +1,237 @@
+"""Convert parsed HTML documents to Markdown."""
+
+import re
+from typing import Any, List, Optional, Union
+
+from .core import BLOCK_TAGS, Node, ParsedDocument, parse_raw
+
+_SKIP_TAGS = {
+    'script', 'style', 'noscript', 'template',
+    'head', 'meta', 'link', 'title',
+}
+_HEADING_TAGS = {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
+_CONTAINER_TAGS = BLOCK_TAGS | {'html', 'body', 'figure', 'picture', 'details'}
+
+
+def to_markdown(target: Union[str, bytes, ParsedDocument],
+                config: Optional[Any] = None) -> str:
+    """Convert HTML or an already parsed document to Markdown."""
+    doc = target if isinstance(target, ParsedDocument) else parse_raw(target, config)
+    return document_to_markdown(doc)
+
+
+def document_to_markdown(doc: ParsedDocument) -> str:
+    """Render a parsed document as Markdown."""
+    blocks: List[str] = []
+    _render_children(doc.root, blocks)
+    return '\n\n'.join(block for block in blocks if block.strip())
+
+
+def _render_children(node: Node, out: List[str]) -> None:
+    for child in node.children:
+        if isinstance(child, str):
+            text = _collapse(child)
+            if text.strip():
+                out.append(text.strip())
+        elif not child.is_comment and child.tag not in _SKIP_TAGS:
+            _render_block(child, out)
+
+
+def _render_block(node: Node, out: List[str]) -> None:
+    tag = node.tag
+    if tag in _HEADING_TAGS:
+        text = _inline(node).strip()
+        if text:
+            out.append('#' * int(tag[1]) + ' ' + text)
+    elif tag == 'p':
+        text = _inline(node).strip()
+        if text:
+            out.append(text)
+    elif tag == 'hr':
+        out.append('---')
+    elif tag == 'pre':
+        out.append(_fenced_code(node))
+    elif tag == 'blockquote':
+        inner: List[str] = []
+        _render_children(node, inner)
+        joined = '\n\n'.join(inner)
+        out.append('\n'.join('> ' + line for line in joined.split('\n')))
+    elif tag in ('ul', 'ol'):
+        out.append(_list(node, 0))
+    elif tag == 'table':
+        text = _markdown_table(node)
+        if text:
+            out.append(text)
+    elif tag in _CONTAINER_TAGS:
+        _render_children(node, out)
+    else:
+        parts: List[str] = []
+        _inline_node(node, parts)
+        text = ''.join(parts).strip()
+        if text:
+            out.append(text)
+
+
+def _inline(node: Node) -> str:
+    parts: List[str] = []
+    _inline_node(node, parts)
+    return ''.join(parts)
+
+
+def _inline_node(node: Node, parts: List[str]) -> None:
+    tag = node.tag
+    if tag == 'br':
+        parts.append('\n')
+        return
+    if tag == 'img':
+        alt = node.get_attr('alt') or ''
+        src = node.get_attr('src') or ''
+        title = node.get_attr('title')
+        suffix = f' "{title}"' if title else ''
+        parts.append(f'![{alt}]({src}{suffix})')
+        return
+    if tag == 'a':
+        href = node.get_attr('href') or ''
+        title = node.get_attr('title')
+        label = _flat(node).strip()
+        if href:
+            suffix = f' "{title}"' if title else ''
+            parts.append(f'[{label}]({href}{suffix})')
+        else:
+            parts.append(label)
+        return
+    if tag in ('strong', 'b'):
+        inner = _flat(node).strip()
+        if inner:
+            parts.append(f'**{inner}**')
+        return
+    if tag in ('em', 'i'):
+        inner = _flat(node).strip()
+        if inner:
+            parts.append(f'*{inner}*')
+        return
+    if tag in ('del', 's'):
+        inner = _flat(node).strip()
+        if inner:
+            parts.append(f'~~{inner}~~')
+        return
+    if tag == 'code':
+        parts.append(_code_span(node.text_content()))
+        return
+    if tag in ('ul', 'ol'):
+        return
+    for child in node.children:
+        if isinstance(child, str):
+            parts.append(_collapse(child))
+        elif not child.is_comment and child.tag not in _SKIP_TAGS:
+            _inline_node(child, parts)
+
+
+def _flat(node: Node) -> str:
+    parts: List[str] = []
+
+    def walk(current: Node) -> None:
+        for child in current.children:
+            if isinstance(child, str):
+                parts.append(_collapse(child))
+            elif child.tag == 'br':
+                parts.append(' ')
+            elif not child.is_comment and child.tag not in _SKIP_TAGS:
+                walk(child)
+
+    walk(node)
+    return ''.join(parts)
+
+
+def _code_span(text: str) -> str:
+    text = text.strip()
+    if '`' not in text:
+        return f'`{text}`'
+    return f'`` {text} ``'
+
+
+def _fenced_code(node: Node) -> str:
+    language = ''
+    for child in node.child_elements():
+        if child.tag == 'code':
+            match = re.search(
+                r'(?:^|\s)(?:language|lang)-([\w#+.-]+)',
+                child.get_attr('class') or '',
+            )
+            if match:
+                language = match.group(1)
+    content = node.text_content().strip('\n')
+    return f'```{language}\n{content}\n```'
+
+
+def _list(node: Node, depth: int) -> str:
+    ordered = node.tag == 'ol'
+    index = 1
+    if ordered:
+        start = node.get_attr('start')
+        if start and start.lstrip('-').isdigit():
+            index = int(start)
+    lines: List[str] = []
+    for child in node.child_elements():
+        if child.tag != 'li':
+            continue
+        marker = f'{index}. ' if ordered else '- '
+        prefix = '    ' * depth + marker
+        lines.append(prefix + _li_content(child, depth))
+        if ordered:
+            index += 1
+    return '\n'.join(lines)
+
+
+def _li_content(li: Node, depth: int) -> str:
+    parts: List[str] = []
+    tail: List[str] = []
+    for child in li.children:
+        if isinstance(child, str):
+            parts.append(_collapse(child))
+        elif child.is_comment or child.tag in _SKIP_TAGS:
+            continue
+        elif child.tag in ('ul', 'ol'):
+            tail.append(_list(child, depth + 1))
+        elif child.tag == 'p':
+            text = _inline(child).strip()
+            if text:
+                parts.append(text)
+        else:
+            parts.append(_inline(child))
+    head = re.sub(r'[ \t]+', ' ', ''.join(parts)).strip()
+    if tail:
+        return head + '\n' + '\n'.join(tail)
+    return head
+
+
+def _markdown_table(node: Node) -> str:
+    from .extractors import _parse_table
+    data = _parse_table(node)
+    header_rows = data['headers']
+    body_rows = list(data['rows'])
+    if header_rows:
+        header = header_rows[0]
+        body_rows = header_rows[1:] + body_rows
+    elif body_rows:
+        header = body_rows.pop(0)
+    else:
+        return ''
+    all_rows = [header] + body_rows
+    width = max(len(row) for row in all_rows)
+    if width == 0:
+        return ''
+
+    def pad(row: List[str]) -> List[str]:
+        return [str(cell).replace('|', '\\|') for cell in row] + [''] * (width - len(row))
+
+    lines = [
+        '| ' + ' | '.join(pad(header)) + ' |',
+        '|' + '---|' * width,
+    ]
+    lines.extend('| ' + ' | '.join(pad(row)) + ' |' for row in body_rows)
+    return '\n'.join(lines)
+
+
+def _collapse(text: str) -> str:
+    return re.sub(r'\s+', ' ', text)

webreader/sanitizer.py

@@ -0,0 +1,117 @@
+"""Tree-based HTML sanitization driven by parser configuration."""
+
+import re
+from html import escape as _escape
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from .core import Node, ParsedDocument
+
+_DROP_CONTENT_TAGS = {'script', 'style', 'noscript', 'template'}
+
+SAFE_ATTRS = frozenset({
+    'href', 'src', 'alt', 'title', 'class', 'id', 'colspan', 'rowspan',
+    'target', 'rel', 'width', 'height', 'name', 'lang', 'datetime',
+    'cite', 'start', 'reversed', 'value', 'type',
+})
+
+_DANGEROUS_URL_RE = re.compile(r'^\s*(?:javascript|vbscript)\s*:', re.IGNORECASE)
+_SAFE_DATA_URL_RE = re.compile(
+    r'^\s*data:image/(?:png|jpe?g|gif|webp|bmp)\s*;', re.IGNORECASE
+)
+
+_FALLBACK_ALLOWED_TAGS = {'p', 'br', 'b', 'i', 'a', 'ul', 'ol', 'li', 'div', 'span'}
+
+
+def sanitize_document(doc: ParsedDocument,
+                      allowed_tags: Optional[Set[str]] = None) -> Tuple[str, Dict[str, Any]]:
+    """Return (clean_html, removal_report) for a parsed document.
+
+    Elements outside the allowed set are unwrapped (their inner text is
+    kept); script/style-like elements are dropped entirely; attributes are
+    filtered through SAFE_ATTRS, event handlers are removed and dangerous
+    URL schemes (javascript:, vbscript:, unsafe data:) are blocked.
+    """
+    allowed = _resolve_allowed(doc, allowed_tags)
+    report: Dict[str, Any] = {
+        'allowed_tags': sorted(allowed),
+        'removed_tags': {},
+        'removed_attributes': {},
+        'blocked_urls': 0,
+        'removed_comments': 0,
+        'threats_detected': False,
+    }
+    cleaned = _clean_children(doc.root, allowed, report)
+    html = ''.join(
+        _escape(child, quote=False) if isinstance(child, str) else child.to_html()
+        for child in cleaned
+    )
+    return html, report
+
+
+def _resolve_allowed(doc: ParsedDocument,
+                     allowed_tags: Optional[Set[str]]) -> Set[str]:
+    if allowed_tags is not None:
+        return {tag.lower() for tag in allowed_tags}
+    from_config = doc.config.get('allowed_tags')
+    if from_config:
+        return {tag.lower() for tag in from_config}
+    return set(_FALLBACK_ALLOWED_TAGS)
+
+
+_URL_CONTROL_CHARS_RE = re.compile(r'[\t\n\r\f\v\0]')
+
+
+def _is_dangerous_url(url: str) -> bool:
+    url = _URL_CONTROL_CHARS_RE.sub('', url).strip()
+    if _DANGEROUS_URL_RE.match(url):
+        return True
+    if url.lower().startswith('data:'):
+        return not _SAFE_DATA_URL_RE.match(url)
+    return False
+
+
+def _clean_attrs(node: Node, report: Dict[str, Any]) -> Dict[str, str]:
+    attrs: Dict[str, str] = {}
+    for name, value in node.attrs.items():
+        lowered = name.lower()
+        if lowered.startswith('on'):
+            _count(report['removed_attributes'], name)
+            report['threats_detected'] = True
+            continue
+        if lowered not in SAFE_ATTRS:
+            _count(report['removed_attributes'], name)
+            continue
+        if lowered in ('href', 'src') and _is_dangerous_url(value or ''):
+            report['blocked_urls'] += 1
+            report['threats_detected'] = True
+            continue
+        attrs[name] = value
+    return attrs
+
+
+def _clean_children(node: Node, allowed: Set[str],
+                    report: Dict[str, Any]) -> List[Any]:
+    out: List[Any] = []
+    for child in node.children:
+        if isinstance(child, str):
+            out.append(child)
+            continue
+        if child.is_comment:
+            report['removed_comments'] += 1
+            continue
+        if child.tag in _DROP_CONTENT_TAGS:
+            _count(report['removed_tags'], child.tag)
+            report['threats_detected'] = True
+            continue
+        if child.tag not in allowed:
+            _count(report['removed_tags'], child.tag)
+            out.extend(_clean_children(child, allowed, report))
+            continue
+        cleaned = Node(child.tag, _clean_attrs(child, report))
+        cleaned.children = _clean_children(child, allowed, report)
+        out.append(cleaned)
+    return out
+
+
+def _count(counter: Dict[str, int], name: str) -> None:
+    counter[name] = counter.get(name, 0) + 1