webreader 2.3.8__py3-none-any.whl

webreader/__init__.py

@@ -0,0 +1,165 @@
+"""Tree-based HTML parsing, extraction and sanitization."""
+
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+import pafer
+from pafer.exceptions import (
+    ConfigLoadError,
+    HTMLPaferError,
+    SanitizationError,
+    SelectorSyntaxError,
+)
+
+from . import core as _core
+from .core import Engine, Node, ParsedDocument, get_config, get_engine, load_config
+
+def _as_doc(target: Union[str, bytes, ParsedDocument]) -> ParsedDocument:
+    if isinstance(target, ParsedDocument):
+        return target
+    return _core.get_engine().parse(target)
+
+
+def parse(html: Union[str, bytes]) -> ParsedDocument:
+    """Parse HTML into a ParsedDocument using the active configuration."""
+    return _core.get_engine().parse(html)
+
+
+def select(target: Union[str, bytes, ParsedDocument],
+           selector: str) -> List[Dict[str, Any]]:
+    """Run a CSS selector; returns dicts with tag/attrs/text/html per match."""
+    return [node.to_dict() for node in _as_doc(target).find_all(selector)]
+
+
+def extract_metadata(target: Union[str, bytes, ParsedDocument]) -> Dict[str, Any]:
+    """Extract title, OG/Twitter, canonical, lang, charset and JSON-LD."""
+    return _as_doc(target).metadata()
+
+
+def extract_links(target: Union[str, bytes, ParsedDocument],
+                  base_url: Optional[str] = None) -> List[Dict[str, Any]]:
+    """Extract links and asset references, absolutized against base_url."""
+    return _as_doc(target).links(base_url=base_url)
+
+
+def extract_tables(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Extract tables as header/data matrices with span expansion."""
+    return _as_doc(target).tables()
+
+
+def extract_text(target: Union[str, bytes, ParsedDocument]) -> str:
+    """Extract readable text with boilerplate removal."""
+    return _as_doc(target).text()
+
+
+def sanitize(target: Union[str, bytes, ParsedDocument],
+             allowed_tags: Optional[Set[str]] = None) -> Tuple[str, Dict[str, Any]]:
+    """Return (clean_html, removal_report) using config allowed_tags by default."""
+    from .sanitizer import sanitize_document
+    return sanitize_document(_as_doc(target), allowed_tags=allowed_tags)
+
+
+def minify(html: str) -> str:
+    """Minify HTML honoring remove_comments from the active configuration."""
+    return _core.get_engine().minify(html)
+
+
+def pretty(html: Union[str, bytes], indent: str = '  ') -> str:
+    """Reformat HTML into indented, readable markup."""
+    from .formatter import pretty_html
+    return pretty_html(html, indent=indent, config=_core.config)
+
+
+def to_markdown(target: Union[str, bytes, ParsedDocument]) -> str:
+    """Convert HTML or a parsed document to Markdown."""
+    from .markdown import to_markdown as _to_md
+    return _to_md(target, config=_core.config)
+
+def extract_images(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Extract img elements with src, alt, dimensions and srcset candidates."""
+    return _as_doc(target).images()
+
+
+def extract_forms(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Extract forms with their input fields and default values."""
+    return _as_doc(target).forms()
+
+
+def extract_headings(target: Union[str, bytes, ParsedDocument]) -> Dict[str, Any]:
+    """Extract the h1-h6 outline plus word count and reading time."""
+    return _as_doc(target).headings()
+
+
+def extract_scripts(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Inventory script elements, both inline and external."""
+    return _as_doc(target).scripts()
+
+
+def extract_stylesheets(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Inventory stylesheets: link[rel~=stylesheet] and inline styles."""
+    return _as_doc(target).stylesheets()
+
+
+def export(data: Any, fmt: str = 'json', indent: int = 2) -> str:
+    """Export data to json, csv or markdown using pafer.exporters."""
+    if fmt == 'json':
+        return pafer.exporters.DataExporter.to_json(data, indent=indent)
+    if fmt == 'csv':
+        if isinstance(data, list) and data and all(isinstance(row, dict) for row in data):
+            headers = list(data[0].keys())
+            rows = [[str(row.get(header, '')) for header in headers] for row in data]
+            return pafer.exporters.DataExporter.to_csv([headers] + rows)
+        return pafer.exporters.DataExporter.to_csv(data)
+    if fmt == 'md':
+        return _export_markdown(data)
+    raise ValueError(f'unsupported export format: {fmt!r}')
+
+
+def _export_markdown(data: Any) -> str:
+    if isinstance(data, list) and data and all(isinstance(item, dict) and 'rows' in item
+                                               for item in data):
+        lines: List[str] = []
+        for table in data:
+            matrix = list(table.get('headers', [])) + list(table.get('rows', []))
+            width = max((len(row) for row in matrix), default=0)
+            if not width:
+                continue
+            padded = [list(row) + [''] * (width - len(row)) for row in matrix]
+            lines.append('| ' + ' | '.join(str(cell) for cell in padded[0]) + ' |')
+            lines.append('|' + '---|' * width)
+            lines.extend(
+                '| ' + ' | '.join(str(cell) for cell in row) + ' |'
+                for row in padded[1:]
+            )
+            lines.append('')
+        return '\n'.join(lines)
+    return '```json\n' + pafer.exporters.DataExporter.to_json(data) + '\n```'
+
+
+__all__ = [
+    'Engine',
+    'Node',
+    'ParsedDocument',
+    'load_config',
+    'get_config',
+    'get_engine',
+    'parse',
+    'select',
+    'extract_metadata',
+    'extract_links',
+    'extract_tables',
+    'extract_text',
+    'sanitize',
+    'minify',
+    'pretty',
+    'to_markdown',
+    'extract_images',
+    'extract_forms',
+    'extract_headings',
+    'extract_scripts',
+    'extract_stylesheets',
+    'export',
+    'HTMLPaferError',
+    'ConfigLoadError',
+    'SanitizationError',
+    'SelectorSyntaxError',
+]

webreader/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running the package as `python -m webreader`."""
+
+from .cli import run
+
+if __name__ == '__main__':
+    run()

webreader/cli.py

@@ -0,0 +1,166 @@
+"""Command line interface for webreader."""
+
+import argparse
+import json
+import sys
+from typing import Any, Dict, List, Optional
+
+from pafer.exporters import DataExporter
+
+
+def run(argv: Optional[List[str]] = None) -> None:
+    """Entry point for the webreader CLI."""
+    cli = argparse.ArgumentParser(
+        prog='webreader',
+        description='webreader - HTML processing tool built on the reader package',
+        epilog="Example: python -m webreader parse -f input.html",
+    )
+    cli.add_argument('--config', help='load this config.config as the new default configuration')
+    sub = cli.add_subparsers(dest='command', required=True)
+
+    def with_file(cmd):
+        cmd.add_argument('-f', '--file', required=True, help="Path to HTML file ('-' for stdin)")
+        return cmd
+
+    def with_out(cmd):
+        cmd.add_argument('--out', default=None, help='Write output to this file')
+        return cmd
+
+    with_out(with_file(sub.add_parser('parse', help='Parse a file and print the structured result')))
+
+    select_cmd = with_out(with_file(sub.add_parser('select', help='Run a CSS selector and print matches')))
+    select_cmd.add_argument('-s', '--selector', required=True, help='CSS selector expression')
+
+    links_cmd = with_out(with_file(sub.add_parser('links', help='List links and asset references')))
+    links_cmd.add_argument('--base-url', default=None, help='Absolutize URLs against this base')
+
+    with_out(with_file(sub.add_parser('meta', help='Extract page metadata')))
+
+    tables_cmd = with_out(with_file(sub.add_parser('tables', help='Extract tables')))
+    tables_cmd.add_argument('--format', choices=['json', 'csv', 'md'], default='json')
+
+    with_out(with_file(sub.add_parser('text', help='Extract readable text')))
+
+    with_out(with_file(sub.add_parser('images', help='List images')))
+
+    with_out(with_file(sub.add_parser('forms', help='Extract forms and fields')))
+
+    with_out(with_file(sub.add_parser('minify', help='Minify HTML')))
+
+    with_out(with_file(sub.add_parser('pretty', help='Reformat HTML with indentation')))
+
+    with_out(with_file(sub.add_parser('md', help='Convert HTML to Markdown')))
+
+    sanitize_cmd = with_out(with_file(sub.add_parser('sanitize', help='Sanitize HTML and print or save the result')))
+    sanitize_cmd.add_argument('--allowed', default=None, help='Comma-separated allowed tags')
+
+    args = cli.parse_args(argv)
+
+    import webreader
+
+    if args.config:
+        try:
+            webreader.load_config(args.config)
+            print(f"[+] Loaded config from '{args.config}' (persisted as webreader default).")
+        except Exception as exc:
+            print(f'[-] Config error: {exc}')
+            sys.exit(1)
+
+    try:
+        html_text = _read_input(args.file)
+    except OSError as exc:
+        print(f"[-] Error: cannot read '{args.file}': {exc}")
+        sys.exit(1)
+
+    try:
+        if args.command == 'parse':
+            _output(_dumps(webreader.parse(html_text).to_dict()), args.out)
+        elif args.command == 'select':
+            _output(_dumps(webreader.select(html_text, args.selector)), args.out)
+        elif args.command == 'links':
+            result = webreader.extract_links(html_text, base_url=args.base_url)
+            _output(_dumps(result), args.out)
+        elif args.command == 'meta':
+            _output(_dumps(webreader.extract_metadata(html_text)), args.out)
+        elif args.command == 'tables':
+            _output(_tables_output(webreader.extract_tables(html_text), args.format), args.out)
+        elif args.command == 'text':
+            _output(webreader.extract_text(html_text), args.out)
+        elif args.command == 'images':
+            _output(_dumps(webreader.extract_images(html_text)), args.out)
+        elif args.command == 'forms':
+            _output(_dumps(webreader.extract_forms(html_text)), args.out)
+        elif args.command == 'minify':
+            _output(webreader.minify(html_text), args.out)
+        elif args.command == 'pretty':
+            _output(webreader.pretty(html_text), args.out)
+        elif args.command == 'md':
+            _output(webreader.to_markdown(html_text), args.out)
+        elif args.command == 'sanitize':
+            allowed: Optional[List[str]] = None
+            if args.allowed:
+                allowed = [tag.strip() for tag in args.allowed.split(',') if tag.strip()]
+            clean, report = webreader.sanitize(html_text, allowed_tags=allowed)
+            if args.out:
+                _write_file(args.out, clean)
+                print(f"[+] Sanitized HTML written to '{args.out}'.")
+            else:
+                print(clean)
+            print('[+] Report: ' + json.dumps(report, default=str))
+    except Exception as exc:
+        print(f'[-] Error: {exc}')
+        sys.exit(1)
+
+
+def _read_input(path: str) -> str:
+    if path == '-':
+        return sys.stdin.read()
+    with open(path, 'r', encoding='utf-8-sig', errors='replace') as handle:
+        return handle.read()
+
+
+def _write_file(path: str, text: str) -> None:
+    with open(path, 'w', encoding='utf-8') as handle:
+        handle.write(text)
+
+
+def _output(text: str, out: Optional[str]) -> None:
+    if out:
+        _write_file(out, text if text.endswith('\n') else text + '\n')
+        print(f"[+] Written to '{out}'.")
+    else:
+        print(text)
+
+
+def _dumps(data: Any) -> str:
+    return json.dumps(data, indent=2, default=str)
+
+
+def _tables_output(tables: List[Dict[str, Any]], fmt: str) -> str:
+    if fmt == 'json':
+        return _dumps(tables)
+    rows: List[List[str]] = []
+    for table in tables:
+        rows.extend(table.get('headers', []))
+        rows.extend(table.get('rows', []))
+        rows.append([])
+    if not rows:
+        return ''
+    if fmt == 'csv':
+        return DataExporter.to_csv(rows).rstrip('\n')
+    return _markdown_table(rows)
+
+
+def _markdown_table(rows: List[List[str]]) -> str:
+    rows = [row for row in rows if row]
+    if not rows:
+        return ''
+    width = max(len(row) for row in rows)
+    padded = [list(row) + [''] * (width - len(row)) for row in rows]
+    lines = ['| ' + ' | '.join(padded[0]) + ' |', '|' + '---|' * width]
+    lines.extend('| ' + ' | '.join(row) + ' |' for row in padded[1:])
+    return '\n'.join(lines)
+
+
+if __name__ == '__main__':
+    run()