webreader 2.3.8__py3-none-any.whl

webreader/extractors.py

@@ -0,0 +1,447 @@
+"""Content extractors: metadata, links/assets, tables and readable text."""
+
+import json
+import re
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+from urllib.parse import urljoin
+
+from .core import BLOCK_TAGS, Node, ParsedDocument
+
+_SKIP_TEXT_TAGS = {'script', 'style', 'noscript', 'template', 'head'}
+_BOILERPLATE_TAGS = {'nav', 'footer'}
+_BOILERPLATE_NAME_RE = re.compile(
+    r'\b(nav|navbar|menu|sidebar|header|footer|banner|advert|ads?|promo|breadcrumb)\b',
+    re.IGNORECASE,
+)
+
+_URL_ATTRS = {
+    'a': 'href',
+    'img': 'src',
+    'script': 'src',
+    'iframe': 'src',
+    'source': 'src',
+    'embed': 'src',
+    'track': 'src',
+    'link': 'href',
+    'area': 'href',
+}
+
+_SIMPLE_META_KEYS = (
+    'description', 'author', 'keywords', 'viewport', 'generator', 'robots',
+)
+
+_HEADING_TAGS = {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
+
+_WORDS_PER_MINUTE = 200
+
+
+def _parse_srcset(value: str) -> List[Tuple[str, str]]:
+    """Split a srcset attribute into (url, descriptor) candidates."""
+    candidates: List[Tuple[str, str]] = []
+    for part in value.split(','):
+        tokens = part.strip().split()
+        if tokens:
+            candidates.append((tokens[0], tokens[1] if len(tokens) > 1 else ''))
+    return candidates
+
+
+def _first(root: Node, tag: str) -> Optional[Node]:
+    for element in root.iter_elements():
+        if element.tag == tag:
+            return element
+    return None
+
+
+def _meta_entries(root: Node) -> Iterator[Tuple[str, str]]:
+    for element in root.iter_elements():
+        if element.tag == 'meta':
+            key = element.get_attr('property') or element.get_attr('name')
+            if key:
+                yield key.strip().lower(), (element.get_attr('content') or '')
+
+
+def extract_metadata(doc: ParsedDocument) -> Dict[str, Any]:
+    """Extract title, description, Open Graph, Twitter card, canonical URL,
+    lang, charset and JSON-LD blocks from a parsed document."""
+    meta: Dict[str, Any] = {}
+    open_graph: Dict[str, str] = {}
+    twitter: Dict[str, str] = {}
+
+    for key, content in _meta_entries(doc.root):
+        if key.startswith('og:'):
+            open_graph[key[3:]] = content
+        elif key.startswith('twitter:'):
+            twitter[key[8:]] = content
+        elif key in _SIMPLE_META_KEYS:
+            meta[key] = content
+
+    title = _first(doc.root, 'title')
+    if title is not None:
+        meta['title'] = title.text_content().strip()
+
+    html_el = _first(doc.root, 'html')
+    if html_el is not None and html_el.get_attr('lang'):
+        meta['lang'] = html_el.get_attr('lang')
+
+    for element in doc.root.iter_elements():
+        if element.tag == 'link':
+            rel = (element.get_attr('rel') or '').lower()
+            if rel == 'canonical' and element.get_attr('href'):
+                meta['canonical'] = element.get_attr('href')
+                break
+
+    for element in doc.root.iter_elements():
+        if element.tag != 'meta':
+            continue
+        charset = element.get_attr('charset')
+        if charset:
+            meta['charset'] = charset.lower()
+            break
+        if (element.get_attr('http-equiv') or '').lower() == 'content-type':
+            match = re.search(
+                r'charset=([\w-]+)', element.get_attr('content') or '', re.IGNORECASE
+            )
+            if match:
+                meta['charset'] = match.group(1).lower()
+                break
+
+    json_ld: List[Any] = []
+    for element in doc.root.iter_elements():
+        if element.tag == 'script':
+            type_ = (element.get_attr('type') or '').lower()
+            if type_ == 'application/ld+json':
+                raw = element.text_content().strip()
+                try:
+                    json_ld.append(json.loads(raw))
+                except ValueError:
+                    json_ld.append(raw)
+    if json_ld:
+        meta['json_ld'] = json_ld
+    if open_graph:
+        meta['open_graph'] = open_graph
+    if twitter:
+        meta['twitter'] = twitter
+    return meta
+
+
+def extract_links(doc: ParsedDocument,
+                  base_url: Optional[str] = None) -> List[Dict[str, Any]]:
+    """Extract hyperlinks and asset references, optionally absolutized
+    against base_url."""
+    links: List[Dict[str, Any]] = []
+
+    def add(element: Node, url: str, kind: str) -> None:
+        entry: Dict[str, Any] = {
+            'tag': element.tag,
+            'kind': kind,
+            'url': urljoin(base_url, url) if base_url else url,
+        }
+        if element.tag == 'a':
+            entry['text'] = element.text_content().strip()[:200]
+        if element.tag == 'img':
+            entry['alt'] = element.get_attr('alt') or ''
+        if element.tag == 'link':
+            entry['rel'] = (element.get_attr('rel') or '').lower()
+        links.append(entry)
+
+    for element in doc.root.iter_elements():
+        attr = _URL_ATTRS.get(element.tag)
+        if attr:
+            url = element.get_attr(attr)
+            if url:
+                add(element, url, 'href' if attr == 'href' else 'src')
+        srcset = element.get_attr('srcset')
+        if srcset:
+            for url, _ in _parse_srcset(srcset):
+                add(element, url, 'srcset')
+    return links
+
+
+def _int_attr(cell: Node, name: str) -> int:
+    try:
+        return max(1, int(cell.get_attr(name) or 1))
+    except (TypeError, ValueError):
+        return 1
+
+
+def _in_thead(row: Node) -> bool:
+    node = row.parent
+    while node is not None:
+        if node.tag == 'thead':
+            return True
+        if node.tag == 'table':
+            return False
+        node = node.parent
+    return False
+
+
+def _table_rows(table: Node) -> List[Node]:
+    rows: List[Node] = []
+
+    def walk(node: Node) -> None:
+        for child in node.children:
+            if not isinstance(child, Node) or child.is_comment:
+                continue
+            if child.tag == 'tr':
+                rows.append(child)
+            elif child.tag in ('thead', 'tbody', 'tfoot'):
+                walk(child)
+
+    walk(table)
+    return rows
+
+
+def _row_cells(row: Node) -> List[Node]:
+    return [child for child in row.child_elements() if child.tag in ('td', 'th')]
+
+
+def _parse_table(table: Node) -> Dict[str, Any]:
+    caption = ''
+    for child in table.child_elements():
+        if child.tag == 'caption':
+            caption = child.text_content().strip()
+            break
+
+    rows = _table_rows(table)
+    grid: Dict[Tuple[int, int], str] = {}
+    max_col = 0
+    for r, row in enumerate(rows):
+        c = 0
+        for cell in _row_cells(row):
+            while (r, c) in grid:
+                c += 1
+            colspan = _int_attr(cell, 'colspan')
+            rowspan = _int_attr(cell, 'rowspan')
+            text = cell.text_content().strip()
+            for dr in range(rowspan):
+                for dc in range(colspan):
+                    grid[(r + dr, c + dc)] = text
+            c += colspan
+            if c > max_col:
+                max_col = c
+
+    width = max_col
+    height = (max(r for r, _ in grid) + 1) if grid else 0
+    matrix = [[grid.get((r, c), '') for c in range(width)] for r in range(height)]
+
+    header_indexes = [i for i, row in enumerate(rows) if _in_thead(row)]
+    if not header_indexes and rows and _row_cells(rows[0]):
+        if all(cell.tag == 'th' for cell in _row_cells(rows[0])):
+            header_indexes = [0]
+
+    headers = [matrix[i] for i in header_indexes if i < height]
+    header_set = set(header_indexes)
+    data = [matrix[i] for i in range(height) if i not in header_set]
+
+    return {
+        'caption': caption,
+        'headers': headers,
+        'rows': data,
+        'row_count': len(data),
+        'col_count': width,
+    }
+
+
+def extract_tables(doc: ParsedDocument) -> List[Dict[str, Any]]:
+    """Extract tables into header/data matrices with colspan/rowspan expansion."""
+    tables: List[Dict[str, Any]] = []
+    for element in doc.root.iter_elements():
+        if element.tag == 'table':
+            tables.append(_parse_table(element))
+    return tables
+
+
+def extract_images(doc: ParsedDocument) -> List[Dict[str, Any]]:
+    """Extract img elements with src, alt, dimensions and srcset candidates."""
+    images: List[Dict[str, Any]] = []
+    for element in doc.root.iter_elements():
+        if element.tag != 'img':
+            continue
+        srcset = element.get_attr('srcset')
+        images.append({
+            'src': element.get_attr('src') or '',
+            'alt': element.get_attr('alt') or '',
+            'width': element.get_attr('width') or '',
+            'height': element.get_attr('height') or '',
+            'loading': element.get_attr('loading') or '',
+            'srcset': _parse_srcset(srcset) if srcset else [],
+        })
+    return images
+
+
+def _input_field(node: Node) -> Dict[str, Any]:
+    return {
+        'tag': 'input',
+        'type': (node.get_attr('type') or 'text').lower(),
+        'name': node.get_attr('name') or '',
+        'value': node.get_attr('value') or '',
+        'placeholder': node.get_attr('placeholder') or '',
+        'checked': node.get_attr('checked') is not None,
+        'required': node.get_attr('required') is not None,
+    }
+
+
+def _select_field(node: Node) -> Dict[str, Any]:
+    options: List[Dict[str, Any]] = []
+    for child in node.iter_elements():
+        if child.tag != 'option':
+            continue
+        value = child.get_attr('value')
+        options.append({
+            'value': value if value is not None else child.text_content().strip(),
+            'text': child.text_content().strip(),
+            'selected': child.get_attr('selected') is not None,
+        })
+    return {
+        'tag': 'select',
+        'name': node.get_attr('name') or '',
+        'multiple': node.get_attr('multiple') is not None,
+        'required': node.get_attr('required') is not None,
+        'options': options,
+    }
+
+
+def _textarea_field(node: Node) -> Dict[str, Any]:
+    return {
+        'tag': 'textarea',
+        'name': node.get_attr('name') or '',
+        'value': node.text_content().strip(),
+        'placeholder': node.get_attr('placeholder') or '',
+        'required': node.get_attr('required') is not None,
+    }
+
+
+def _button_field(node: Node) -> Dict[str, Any]:
+    return {
+        'tag': 'button',
+        'type': (node.get_attr('type') or 'submit').lower(),
+        'name': node.get_attr('name') or '',
+        'value': node.get_attr('value') or '',
+        'text': node.text_content().strip(),
+    }
+
+
+_FIELD_HANDLERS = {
+    'input': _input_field,
+    'select': _select_field,
+    'textarea': _textarea_field,
+    'button': _button_field,
+}
+
+
+def extract_forms(doc: ParsedDocument) -> List[Dict[str, Any]]:
+    """Extract forms with their input fields and default values."""
+    forms: List[Dict[str, Any]] = []
+    for element in doc.root.iter_elements():
+        if element.tag != 'form':
+            continue
+        form: Dict[str, Any] = {
+            'action': element.get_attr('action') or '',
+            'method': (element.get_attr('method') or 'get').lower(),
+            'enctype': element.get_attr('enctype')
+                       or 'application/x-www-form-urlencoded',
+            'fields': [],
+        }
+        for field in element.iter_elements():
+            handler = _FIELD_HANDLERS.get(field.tag)
+            if handler:
+                form['fields'].append(handler(field))
+        forms.append(form)
+    return forms
+
+
+def extract_headings(doc: ParsedDocument) -> Dict[str, Any]:
+    """Extract the h1-h6 outline plus word count and reading time."""
+    headings: List[Dict[str, Any]] = []
+    for element in doc.root.iter_elements():
+        if element.tag in _HEADING_TAGS:
+            headings.append({
+                'level': int(element.tag[1]),
+                'text': element.text_content().strip(),
+            })
+    words = len(extract_text(doc).split())
+    return {
+        'headings': headings,
+        'word_count': words,
+        'reading_time_minutes': round(words / _WORDS_PER_MINUTE, 1),
+    }
+
+
+def extract_scripts(doc: ParsedDocument) -> List[Dict[str, Any]]:
+    """Inventory script elements, both inline and external."""
+    scripts: List[Dict[str, Any]] = []
+    for element in doc.root.iter_elements():
+        if element.tag != 'script':
+            continue
+        src = element.get_attr('src')
+        entry: Dict[str, Any] = {
+            'src': src or '',
+            'type': (element.get_attr('type') or '').lower(),
+            'async': element.get_attr('async') is not None,
+            'defer': element.get_attr('defer') is not None,
+            'inline': src is None,
+        }
+        if src is None:
+            entry['content'] = element.text_content().strip()
+        scripts.append(entry)
+    return scripts
+
+
+def extract_stylesheets(doc: ParsedDocument) -> List[Dict[str, Any]]:
+    """Inventory stylesheets: link[rel~=stylesheet] and inline style elements."""
+    sheets: List[Dict[str, Any]] = []
+    for element in doc.root.iter_elements():
+        if element.tag == 'link':
+            rel = (element.get_attr('rel') or '').lower().split()
+            if 'stylesheet' in rel:
+                sheets.append({
+                    'href': element.get_attr('href') or '',
+                    'media': element.get_attr('media') or '',
+                    'inline': False,
+                })
+        elif element.tag == 'style':
+            sheets.append({
+                'media': element.get_attr('media') or '',
+                'inline': True,
+                'content': element.text_content().strip(),
+            })
+    return sheets
+
+
+def _is_boilerplate(node: Node) -> bool:
+    if node.tag in _BOILERPLATE_TAGS:
+        return True
+    identity = f"{node.get_attr('class') or ''} {node.get_attr('id') or ''}"
+    return bool(_BOILERPLATE_NAME_RE.search(identity))
+
+
+def extract_text(doc: ParsedDocument) -> str:
+    """Extract readable text, dropping scripts/styles and common boilerplate
+    elements while preserving block-level line structure."""
+    parts: List[str] = []
+
+    def walk(node: Node) -> None:
+        for child in node.children:
+            if isinstance(child, str):
+                parts.append(child)
+                continue
+            if child.is_comment:
+                continue
+            if child.tag in _SKIP_TEXT_TAGS or _is_boilerplate(child):
+                continue
+            if child.tag == 'br':
+                parts.append('\n')
+                continue
+            block = child.tag in BLOCK_TAGS
+            if block:
+                parts.append('\n')
+            walk(child)
+            if block:
+                parts.append('\n')
+
+    walk(doc.root)
+    text = ''.join(parts)
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r' ?\n ?', '\n', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()

webreader/formatter.py

@@ -0,0 +1,89 @@
+"""HTML pretty-printer producing indented, readable markup."""
+
+import re
+from html import escape as _escape
+from typing import Any, List, Optional, Union
+
+from .core import (
+    BLOCK_TAGS,
+    VOID_ELEMENTS,
+    ParsedDocument,
+    Node,
+    parse_raw,
+)
+
+_BLOCKISH = BLOCK_TAGS | {'html', 'head', 'body'}
+
+
+def pretty_html(html: Union[str, bytes, ParsedDocument], indent: str = '  ',
+                config: Optional[Any] = None) -> str:
+    """Reformat HTML with one block element per line and indented nesting."""
+    doc = html if isinstance(html, ParsedDocument) else parse_raw(html, config)
+    lines: List[str] = []
+    if doc.doctype:
+        lines.append(doc.doctype)
+    for child in doc.root.children:
+        if isinstance(child, str):
+            text = _collapse(child).strip()
+            if text:
+                lines.append(_escape(text, quote=False))
+        elif child.is_comment:
+            lines.append(f'<!--{child.text_content()}-->')
+        else:
+            lines.extend(_render(child, 0, indent))
+    return '\n'.join(lines)
+
+
+def _render(node: Node, level: int, indent: str) -> List[str]:
+    pad = indent * level
+    inner = indent * (level + 1)
+    opening = _opening_tag(node)
+    if node.tag in VOID_ELEMENTS:
+        return [pad + opening]
+    if node.tag in ('pre', 'textarea'):
+        return [pad + node.to_html()]
+
+    parts: List[Any] = []
+    for child in node.children:
+        if isinstance(child, str):
+            parts.append(('inline', _escape(_collapse(child), quote=False)))
+        elif child.is_comment:
+            parts.append(('inline', f'<!--{child.text_content()}-->'))
+        elif child.tag in _BLOCKISH:
+            parts.append(('block', child))
+        else:
+            parts.append(('inline', child.to_html()))
+
+    has_blocks = any(kind == 'block' for kind, _ in parts)
+    if not has_blocks:
+        text = ''.join(text for kind, text in parts).strip()
+        return [pad + opening + text + f'</{node.tag}>']
+
+    lines = [pad + opening]
+    buffer: List[Any] = []
+    for kind, value in parts:
+        if kind == 'inline':
+            buffer.append(value)
+            continue
+        text = ''.join(buffer).strip()
+        buffer = []
+        if text:
+            lines.append(inner + text)
+        lines.extend(_render(value, level + 1, indent))
+    text = ''.join(buffer).strip()
+    if text:
+        lines.append(inner + text)
+    lines.append(pad + f'</{node.tag}>')
+    return lines
+
+
+def _opening_tag(node: Node) -> str:
+    attrs = ''.join(
+        f' {name}' if value == '' else f' {name}="{_escape(value, quote=True)}"'
+        for name, value in node.attrs.items()
+    )
+    return f'<{node.tag}{attrs}>'
+
+
+def _collapse(text: str) -> str:
+    return re.sub(r'\s+', ' ', text)