webreader 2.3.8__tar.gz

webreader-2.3.8/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 webreader
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

webreader-2.3.8/PKG-INFO

@@ -0,0 +1,188 @@
+Metadata-Version: 2.4
+Name: webreader
+Version: 2.3.8
+Summary: Tree-based HTML reader for Python.
+Author-email: morichan <morichan@gmail.com>
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: User Interfaces
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pafer
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Dynamic: license-file
+
+![Python](https://img.shields.io/badge/python-3.8%2B-blue)
+![Dependencies](https://img.shields.io/badge/dependencies-none-success)
+![Tests](https://img.shields.io/badge/tests-pytest-informational)
+
+## Features
+
+- Tree-based parsing into an editable document tree
+- CSS selectors: combinators, attributes, pseudo-classes, selector lists
+- Extractors: metadata, links, tables, text, images, forms, headings,
+  scripts, stylesheets
+- Sanitization with a removal report
+- Minify, pretty-print, Markdown conversion
+- JSON / CSV / Markdown export
+- CLI tool
+- Standard library only at runtime
+
+## Installation
+
+```
+pip install webreader
+```
+
+## Quickstart
+
+```python
+import webreader
+
+html = '<h1 class="title">Hello</h1><a href="/x">link</a>'
+doc = webreader.parse(html)
+
+webreader.select(doc, 'h1.title')   # [{'tag': ..., 'attrs': ..., 'text': ..., 'html': ...}]
+doc.find('a').get_attr('href')       # '/x'
+
+webreader.minify(html)              # collapsed whitespace
+webreader.pretty(html)              # re-indented, readable markup
+webreader.to_markdown(html)         # convert to Markdown
+
+clean, report = webreader.sanitize(html)
+```
+
+Bytes input is decoded using detected encoding: BOM sniffing first,
+then `<meta charset>` in the first 2 KB, then the configured
+`encoding` fallback.
+
+In non-strict mode, malformed HTML does not raise; the warnings are
+available on the document:
+
+```python
+doc = webreader.parse('<div><p>x</p>')   # unclosed <div>
+doc.errors                                 # ['unclosed tag(s) at end of input: <div>']
+```
+
+## Extraction
+
+```python
+webreader.extract_metadata(html)    # title, OG/Twitter, canonical, charset, JSON-LD
+webreader.extract_links(html)       # links and asset references
+webreader.extract_tables(html)      # matrices with colspan/rowspan expanded
+webreader.extract_text(html)        # readable text without boilerplate
+webreader.extract_images(html)      # src, alt, dimensions, srcset candidates
+webreader.extract_forms(html)       # form fields with types and defaults
+webreader.extract_headings(html)    # h1-h6 outline + word count + reading time
+webreader.extract_scripts(html)     # inline and external script inventory
+webreader.extract_stylesheets(html) # link[rel~=stylesheet] and inline styles
+```
+
+## CSS selectors
+
+Supported syntax: type selectors, `*`, `#id`, `.class`, attribute
+selectors (`[attr]`, `[attr=value]` and the `^=`, `$=`, `*=`, `~=`,
+`|=` operators), the pseudo-classes `:first-child`, `:last-child`,
+`:nth-child(an+b)`, `:not(compound)` and `:contains(text)`, compounds
+such as `div.content#main[href="x"]`, the combinators whitespace
+(descendant), `>` (child), `+` (adjacent sibling) and `~` (general
+sibling), and comma-separated selector lists.
+
+```python
+webreader.select(doc, 'a[href^="https://"]')
+webreader.select(doc, 'ul > li:nth-child(2n+1):not(.skip)')
+webreader.select(doc, 'h2 + p, blockquote p:contains("note")')
+```
+
+Invalid or unsupported selectors raise `SelectorSyntaxError`.
+
+## Tree editing
+
+Nodes support a mutation API, and the edited tree can be
+re-serialized:
+
+```python
+doc = webreader.parse('<div><a href="/x">link</a></div>')
+link = doc.find('a')
+link.set_attr('rel', 'noopener').remove_attr('class')
+
+new = webreader.Node('p')
+new.children.append('added text')
+doc.find('div').append_child(new)
+
+link.unwrap()        # replace the <a> with its inner text
+doc.to_html()        # '<div>link<p>added text</p></div>'
+```
+
+`Node` also provides `remove()` and `replace_with(*nodes)`.
+
+## Markdown conversion
+
+`to_markdown` renders headings, paragraphs, links, images,
+emphasis/strong/strikethrough, inline and fenced code (with language
+detection from `class="language-..."`), nested lists, blockquotes,
+horizontal rules and tables.
+
+```python
+webreader.to_markdown('<h1>T</h1><p>Hi <b>bold</b></p>')
+# '# T\n\nHi **bold**'
+```
+
+## Sanitization
+
+`sanitize()` returns `(clean_html, report)`:
+
+- tags outside the whitelist are unwrapped, inner content kept
+- `script`, `style`, `noscript`, `template` are dropped entirely
+- attributes filtered through a safe-list, `on*` handlers removed
+- `javascript:`, `vbscript:` and non-image `data:` URLs blocked
+- the report lists removed tags/attributes, blocked URLs and detected threats
+
+## CLI
+
+```
+python -m webreader parse -f input.html
+python -m webreader select -f input.html -s 'a[href]'
+python -m webreader links -f input.html --base-url https://example.com
+python -m webreader meta -f input.html
+python -m webreader tables -f input.html --format csv
+python -m webreader text -f input.html
+python -m webreader images -f input.html
+python -m webreader forms -f input.html
+python -m webreader minify -f input.html --out min.html
+python -m webreader pretty -f input.html --out pretty.html
+python -m webreader md -f input.html --out page.md
+python -m webreader sanitize -f input.html --out clean.html --allowed p,a,ul
+```
+
+Use `-f -` to read from stdin. Every output command accepts `--out`
+to write to a file instead of stdout.
+
+### Default settings
+
+| Key | Default | Purpose |
+|---|---|---|
+| `strip_whitespace` | `True` | collapse whitespace in text nodes |
+| `encoding` | `utf-8` | fallback when bytes input has no detectable encoding |
+| `max_depth` | `100` | maximum element nesting |
+| `preserve_comments` | `False` | keep comments in the tree |
+| `strict_mode` | `False` | raise on malformed HTML |
+| `remove_comments` | `True` | strip comments when minifying |
+| `allowed_tags` | `p, br, b, i, a, ul, ol, li, div, span` | sanitize whitelist |
+| `max_file_size_mb` | `10` | reject oversized input |
+| `cache_size` | `50` | parse cache capacity |
+
+## Exceptions
+
+`HTMLPaferError` (base), `ConfigLoadError`, `SanitizationError`,
+`SelectorSyntaxError`.

webreader-2.3.8/README.md

@@ -0,0 +1,164 @@
+![Python](https://img.shields.io/badge/python-3.8%2B-blue)
+![Dependencies](https://img.shields.io/badge/dependencies-none-success)
+![Tests](https://img.shields.io/badge/tests-pytest-informational)
+
+## Features
+
+- Tree-based parsing into an editable document tree
+- CSS selectors: combinators, attributes, pseudo-classes, selector lists
+- Extractors: metadata, links, tables, text, images, forms, headings,
+  scripts, stylesheets
+- Sanitization with a removal report
+- Minify, pretty-print, Markdown conversion
+- JSON / CSV / Markdown export
+- CLI tool
+- Standard library only at runtime
+
+## Installation
+
+```
+pip install webreader
+```
+
+## Quickstart
+
+```python
+import webreader
+
+html = '<h1 class="title">Hello</h1><a href="/x">link</a>'
+doc = webreader.parse(html)
+
+webreader.select(doc, 'h1.title')   # [{'tag': ..., 'attrs': ..., 'text': ..., 'html': ...}]
+doc.find('a').get_attr('href')       # '/x'
+
+webreader.minify(html)              # collapsed whitespace
+webreader.pretty(html)              # re-indented, readable markup
+webreader.to_markdown(html)         # convert to Markdown
+
+clean, report = webreader.sanitize(html)
+```
+
+Bytes input is decoded using detected encoding: BOM sniffing first,
+then `<meta charset>` in the first 2 KB, then the configured
+`encoding` fallback.
+
+In non-strict mode, malformed HTML does not raise; the warnings are
+available on the document:
+
+```python
+doc = webreader.parse('<div><p>x</p>')   # unclosed <div>
+doc.errors                                 # ['unclosed tag(s) at end of input: <div>']
+```
+
+## Extraction
+
+```python
+webreader.extract_metadata(html)    # title, OG/Twitter, canonical, charset, JSON-LD
+webreader.extract_links(html)       # links and asset references
+webreader.extract_tables(html)      # matrices with colspan/rowspan expanded
+webreader.extract_text(html)        # readable text without boilerplate
+webreader.extract_images(html)      # src, alt, dimensions, srcset candidates
+webreader.extract_forms(html)       # form fields with types and defaults
+webreader.extract_headings(html)    # h1-h6 outline + word count + reading time
+webreader.extract_scripts(html)     # inline and external script inventory
+webreader.extract_stylesheets(html) # link[rel~=stylesheet] and inline styles
+```
+
+## CSS selectors
+
+Supported syntax: type selectors, `*`, `#id`, `.class`, attribute
+selectors (`[attr]`, `[attr=value]` and the `^=`, `$=`, `*=`, `~=`,
+`|=` operators), the pseudo-classes `:first-child`, `:last-child`,
+`:nth-child(an+b)`, `:not(compound)` and `:contains(text)`, compounds
+such as `div.content#main[href="x"]`, the combinators whitespace
+(descendant), `>` (child), `+` (adjacent sibling) and `~` (general
+sibling), and comma-separated selector lists.
+
+```python
+webreader.select(doc, 'a[href^="https://"]')
+webreader.select(doc, 'ul > li:nth-child(2n+1):not(.skip)')
+webreader.select(doc, 'h2 + p, blockquote p:contains("note")')
+```
+
+Invalid or unsupported selectors raise `SelectorSyntaxError`.
+
+## Tree editing
+
+Nodes support a mutation API, and the edited tree can be
+re-serialized:
+
+```python
+doc = webreader.parse('<div><a href="/x">link</a></div>')
+link = doc.find('a')
+link.set_attr('rel', 'noopener').remove_attr('class')
+
+new = webreader.Node('p')
+new.children.append('added text')
+doc.find('div').append_child(new)
+
+link.unwrap()        # replace the <a> with its inner text
+doc.to_html()        # '<div>link<p>added text</p></div>'
+```
+
+`Node` also provides `remove()` and `replace_with(*nodes)`.
+
+## Markdown conversion
+
+`to_markdown` renders headings, paragraphs, links, images,
+emphasis/strong/strikethrough, inline and fenced code (with language
+detection from `class="language-..."`), nested lists, blockquotes,
+horizontal rules and tables.
+
+```python
+webreader.to_markdown('<h1>T</h1><p>Hi <b>bold</b></p>')
+# '# T\n\nHi **bold**'
+```
+
+## Sanitization
+
+`sanitize()` returns `(clean_html, report)`:
+
+- tags outside the whitelist are unwrapped, inner content kept
+- `script`, `style`, `noscript`, `template` are dropped entirely
+- attributes filtered through a safe-list, `on*` handlers removed
+- `javascript:`, `vbscript:` and non-image `data:` URLs blocked
+- the report lists removed tags/attributes, blocked URLs and detected threats
+
+## CLI
+
+```
+python -m webreader parse -f input.html
+python -m webreader select -f input.html -s 'a[href]'
+python -m webreader links -f input.html --base-url https://example.com
+python -m webreader meta -f input.html
+python -m webreader tables -f input.html --format csv
+python -m webreader text -f input.html
+python -m webreader images -f input.html
+python -m webreader forms -f input.html
+python -m webreader minify -f input.html --out min.html
+python -m webreader pretty -f input.html --out pretty.html
+python -m webreader md -f input.html --out page.md
+python -m webreader sanitize -f input.html --out clean.html --allowed p,a,ul
+```
+
+Use `-f -` to read from stdin. Every output command accepts `--out`
+to write to a file instead of stdout.
+
+### Default settings
+
+| Key | Default | Purpose |
+|---|---|---|
+| `strip_whitespace` | `True` | collapse whitespace in text nodes |
+| `encoding` | `utf-8` | fallback when bytes input has no detectable encoding |
+| `max_depth` | `100` | maximum element nesting |
+| `preserve_comments` | `False` | keep comments in the tree |
+| `strict_mode` | `False` | raise on malformed HTML |
+| `remove_comments` | `True` | strip comments when minifying |
+| `allowed_tags` | `p, br, b, i, a, ul, ol, li, div, span` | sanitize whitelist |
+| `max_file_size_mb` | `10` | reject oversized input |
+| `cache_size` | `50` | parse cache capacity |
+
+## Exceptions
+
+`HTMLPaferError` (base), `ConfigLoadError`, `SanitizationError`,
+`SelectorSyntaxError`.

webreader-2.3.8/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=78.0.2", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "webreader"
+version = "2.3.8"
+description = "Tree-based HTML reader for Python."
+readme = "README.md"
+license = "MIT"
+authors = [
+    {name = "morichan", email = "morichan@gmail.com"}
+]
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: User Interfaces",
+]
+
+dependencies = [
+    "pafer"
+]
+
+
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-cov",
+]
+
+[tool.setuptools.package-data]
+webreader = [
+    "*.config"
+]
+
+[tool.setuptools]
+packages = ["webreader"]

webreader-2.3.8/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

webreader-2.3.8/setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup, find_packages
+
+setup(
+    packages=find_packages(),
+    include_package_data=True,
+)

webreader-2.3.8/webreader/__init__.py

@@ -0,0 +1,165 @@
+"""Tree-based HTML parsing, extraction and sanitization."""
+
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+import pafer
+from pafer.exceptions import (
+    ConfigLoadError,
+    HTMLPaferError,
+    SanitizationError,
+    SelectorSyntaxError,
+)
+
+from . import core as _core
+from .core import Engine, Node, ParsedDocument, get_config, get_engine, load_config
+
+def _as_doc(target: Union[str, bytes, ParsedDocument]) -> ParsedDocument:
+    if isinstance(target, ParsedDocument):
+        return target
+    return _core.get_engine().parse(target)
+
+
+def parse(html: Union[str, bytes]) -> ParsedDocument:
+    """Parse HTML into a ParsedDocument using the active configuration."""
+    return _core.get_engine().parse(html)
+
+
+def select(target: Union[str, bytes, ParsedDocument],
+           selector: str) -> List[Dict[str, Any]]:
+    """Run a CSS selector; returns dicts with tag/attrs/text/html per match."""
+    return [node.to_dict() for node in _as_doc(target).find_all(selector)]
+
+
+def extract_metadata(target: Union[str, bytes, ParsedDocument]) -> Dict[str, Any]:
+    """Extract title, OG/Twitter, canonical, lang, charset and JSON-LD."""
+    return _as_doc(target).metadata()
+
+
+def extract_links(target: Union[str, bytes, ParsedDocument],
+                  base_url: Optional[str] = None) -> List[Dict[str, Any]]:
+    """Extract links and asset references, absolutized against base_url."""
+    return _as_doc(target).links(base_url=base_url)
+
+
+def extract_tables(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Extract tables as header/data matrices with span expansion."""
+    return _as_doc(target).tables()
+
+
+def extract_text(target: Union[str, bytes, ParsedDocument]) -> str:
+    """Extract readable text with boilerplate removal."""
+    return _as_doc(target).text()
+
+
+def sanitize(target: Union[str, bytes, ParsedDocument],
+             allowed_tags: Optional[Set[str]] = None) -> Tuple[str, Dict[str, Any]]:
+    """Return (clean_html, removal_report) using config allowed_tags by default."""
+    from .sanitizer import sanitize_document
+    return sanitize_document(_as_doc(target), allowed_tags=allowed_tags)
+
+
+def minify(html: str) -> str:
+    """Minify HTML honoring remove_comments from the active configuration."""
+    return _core.get_engine().minify(html)
+
+
+def pretty(html: Union[str, bytes], indent: str = '  ') -> str:
+    """Reformat HTML into indented, readable markup."""
+    from .formatter import pretty_html
+    return pretty_html(html, indent=indent, config=_core.config)
+
+
+def to_markdown(target: Union[str, bytes, ParsedDocument]) -> str:
+    """Convert HTML or a parsed document to Markdown."""
+    from .markdown import to_markdown as _to_md
+    return _to_md(target, config=_core.config)
+
+def extract_images(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Extract img elements with src, alt, dimensions and srcset candidates."""
+    return _as_doc(target).images()
+
+
+def extract_forms(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Extract forms with their input fields and default values."""
+    return _as_doc(target).forms()
+
+
+def extract_headings(target: Union[str, bytes, ParsedDocument]) -> Dict[str, Any]:
+    """Extract the h1-h6 outline plus word count and reading time."""
+    return _as_doc(target).headings()
+
+
+def extract_scripts(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Inventory script elements, both inline and external."""
+    return _as_doc(target).scripts()
+
+
+def extract_stylesheets(target: Union[str, bytes, ParsedDocument]) -> List[Dict[str, Any]]:
+    """Inventory stylesheets: link[rel~=stylesheet] and inline styles."""
+    return _as_doc(target).stylesheets()
+
+
+def export(data: Any, fmt: str = 'json', indent: int = 2) -> str:
+    """Export data to json, csv or markdown using pafer.exporters."""
+    if fmt == 'json':
+        return pafer.exporters.DataExporter.to_json(data, indent=indent)
+    if fmt == 'csv':
+        if isinstance(data, list) and data and all(isinstance(row, dict) for row in data):
+            headers = list(data[0].keys())
+            rows = [[str(row.get(header, '')) for header in headers] for row in data]
+            return pafer.exporters.DataExporter.to_csv([headers] + rows)
+        return pafer.exporters.DataExporter.to_csv(data)
+    if fmt == 'md':
+        return _export_markdown(data)
+    raise ValueError(f'unsupported export format: {fmt!r}')
+
+
+def _export_markdown(data: Any) -> str:
+    if isinstance(data, list) and data and all(isinstance(item, dict) and 'rows' in item
+                                               for item in data):
+        lines: List[str] = []
+        for table in data:
+            matrix = list(table.get('headers', [])) + list(table.get('rows', []))
+            width = max((len(row) for row in matrix), default=0)
+            if not width:
+                continue
+            padded = [list(row) + [''] * (width - len(row)) for row in matrix]
+            lines.append('| ' + ' | '.join(str(cell) for cell in padded[0]) + ' |')
+            lines.append('|' + '---|' * width)
+            lines.extend(
+                '| ' + ' | '.join(str(cell) for cell in row) + ' |'
+                for row in padded[1:]
+            )
+            lines.append('')
+        return '\n'.join(lines)
+    return '```json\n' + pafer.exporters.DataExporter.to_json(data) + '\n```'
+
+
+__all__ = [
+    'Engine',
+    'Node',
+    'ParsedDocument',
+    'load_config',
+    'get_config',
+    'get_engine',
+    'parse',
+    'select',
+    'extract_metadata',
+    'extract_links',
+    'extract_tables',
+    'extract_text',
+    'sanitize',
+    'minify',
+    'pretty',
+    'to_markdown',
+    'extract_images',
+    'extract_forms',
+    'extract_headings',
+    'extract_scripts',
+    'extract_stylesheets',
+    'export',
+    'HTMLPaferError',
+    'ConfigLoadError',
+    'SanitizationError',
+    'SelectorSyntaxError',
+]

webreader-2.3.8/webreader/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running the package as `python -m webreader`."""
+
+from .cli import run
+
+if __name__ == '__main__':
+    run()