smol-html 0.1.0__py3-none-any.whl

smol_html/__init__.py

@@ -0,0 +1,4 @@
+from smol_html.core import SmolHtmlCleaner
+
+all = ["__version__", "SmolHtmlCleaner"]
+__version__ = "0.1.0"

smol_html/core.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import minify_html
+from bs4 import BeautifulSoup, Tag
+from lxml import html as lxml_html
+from lxml.html.clean import Cleaner
+
+
+# -------------------------
+# Public API
+# -------------------------
+class SmolHtmlCleaner:
+    """
+    Small, dependable HTML cleaner/minifier with sensible defaults.
+
+    Parameters
+    ----------
+    non_text_to_keep : set of str, optional
+        Tags preserved even if textless. Default includes meta/media/table/line-break tags.
+    attr_stop_words : set of str, optional
+        Attribute tokens indicating non-content scaffolding/UX. Default contains common UI tokens.
+    remove_header_lists : bool, optional
+        Prune links/lists inside ``<header>``. Default True.
+    remove_footer_lists : bool, optional
+        Prune links/lists inside ``<footer>``. Default True.
+    minify : bool, optional
+        Minify HTML output via ``minify_html``. Default True.
+    minify_kwargs : dict, optional
+        Extra args for ``minify_html.minify``. Default empty.
+    pre_parse_hooks : sequence of callables, optional
+        Functions ``(str) -> str`` applied before parsing.
+    post_clean_hooks : sequence of callables, optional
+        Functions ``(BeautifulSoup) -> BeautifulSoup`` applied after cleaning.
+    lxml_* : various, optional
+        Direct mapping to ``lxml.html.clean.Cleaner`` kwargs (e.g., ``lxml_comments``, ``lxml_style``).
+
+    Notes
+    -----
+    Defaults and cleaning behavior are preserved; only the configuration surface
+    moved from a dataclass to keyword-only parameters on the constructor.
+    """
+
+    def __init__(
+        self,
+        *,
+        # Core behavior
+        non_text_to_keep: set[str] = None,
+        attr_stop_words: set[str] = None,
+        remove_header_lists: bool = True,
+        remove_footer_lists: bool = True,
+        # Minify
+        minify: bool = True,
+        minify_kwargs: dict | None = None,
+        # lxml Cleaner exposed explicitly (prefixed)
+        meta: bool = False,
+        page_structure: bool = False,
+        links: bool = True,
+        scripts: bool = False,
+        javascript: bool = True,
+        comments: bool = True,
+        style: bool = True,
+        processing_instructions: bool = True,
+        embedded: bool = True,
+        frames: bool = True,
+        forms: bool = True,
+        annoying_tags: bool = True,
+        kill_tags: set[str] | None = None,
+        remove_unknown_tags: bool = True,
+        safe_attrs_only: bool = True,
+        safe_attrs: set[str] = None,
+    ):
+        # Inline defaults identical to the prior CleanerConfig
+        if safe_attrs is None:
+            safe_attrs = {"href", "hreflang", "src", "srclang", "target", "alt", "kind", "type", "role", "abbr",
+            "accept", "accept-charset", "datetime", "lang", "name", "rel", "title", "value", "content", "label",
+            "item_type", "property", "itemprop"}
+
+        if attr_stop_words is None:
+            attr_stop_words = {"alert", "button", "checkbox", "dialog", "navigation", "tab", "tabpanel", "textbox",
+            "menu", "banner", "form", "search", "progressbar", "radio", "slider", "comment", "nav", "sidebar",
+            "breadcrumb", "dropdown", "menu-item", "toggle", "hamburger", "aside", "tooltip", "modal", "overlay",
+            "popup", "advert", "hero", "utility", "login", "signup", "password", "email", "username"}
+
+        if non_text_to_keep is None:
+            non_text_to_keep = {"meta", "img", "picture", "figure", "figcaption", "video", "source", "audio", "table",
+            "tr", "th", "td", "thead", "tbody", "tfoot", "caption", "br"}
+
+        self.non_text_to_keep = non_text_to_keep
+        self.attr_stop_words = attr_stop_words
+        self.remove_header_lists = remove_header_lists
+        self.remove_footer_lists = remove_footer_lists
+        self.minify = minify
+        self.minify_kwargs = dict(minify_kwargs or {})
+
+        # Initialize lxml Cleaner with explicit kwargs gathered from parameters
+        self._cleaner = Cleaner(
+            meta=meta,
+            page_structure=page_structure,
+            links=links,
+            scripts=scripts,
+            javascript=javascript,
+            comments=comments,
+            style=style,
+            processing_instructions=processing_instructions,
+            embedded=embedded,
+            frames=frames,
+            forms=forms,
+            annoying_tags=annoying_tags,
+            kill_tags=kill_tags,
+            remove_unknown_tags=remove_unknown_tags,
+            safe_attrs_only=safe_attrs_only,
+            safe_attrs=safe_attrs,
+        )
+
+    # -------------------------
+    # User-friendly entry points
+    # -------------------------
+
+
+    def clean(self, *, raw_html: str | BeautifulSoup) -> str:
+        """Clean and optionally minify HTML input.
+
+        The cleaning pipeline applies pre-parse hooks (on strings), prunes elements
+        by attribute stop words, sanitizes via lxml Cleaner, performs structural
+        pruning of header/footer/body, then applies post-clean hooks.
+
+        Parameters
+        ----------
+        raw_html : str or BeautifulSoup
+            Raw HTML string or BeautifulSoup to be cleaned.
+
+        Returns
+        -------
+        str
+            Cleaned HTML as a string.
+        """
+
+        # Stage 0: hooks that operate on the raw string
+        if isinstance(raw_html, str):
+            soup = BeautifulSoup(raw_html or "", features="lxml")
+        elif isinstance(raw_html, BeautifulSoup):
+            soup = raw_html
+        else:
+            raise TypeError("raw_html must be a str or BeautifulSoup instance")
+
+        # Stage 1: attribute-based pruning on the original soup
+        # Remove small, likely non-content elements based on attribute tokens.
+        self._strip_by_attribute_stop_words(soup=soup)
+
+        # Stage 2: lxml cleaner pass (robust HTML sanitation)
+        # Use lxml Cleaner to sanitize HTML, optionally minify afterwards.
+        cleaned_html = self._lxml_clean(str(soup))
+        clean_soup = BeautifulSoup(markup=cleaned_html, features="lxml")
+
+        # Stage 3: structural pruning on header/body/footer of the cleaned soup
+        self._prune_header_footer(clean_soup)
+        self._prune_body(clean_soup)
+        self._drop_empty_leaf_nodes(clean_soup)
+
+        return str(clean_soup)
+
+    # -------------------------
+    # Internal helpers
+    # -------------------------
+    def _lxml_clean(self, html_str: str) -> str:
+        """Sanitize and optionally minify HTML using lxml + minify_html.
+
+        Parameters
+        ----------
+        html_str : str
+            HTML markup to be cleaned.
+
+        Returns
+        -------
+        str
+            Cleaned (and possibly minified) HTML markup.
+        """
+        try:
+            cleaned = self._cleaner.clean_html(html_str)
+            return minify_html.minify(cleaned, **self.minify_kwargs) if self.minify else cleaned
+        except ValueError as ex:
+            # Handle encoding declaration edge-cases by round-tripping via lxml
+            msg = (
+                "Unicode strings with encoding declaration are not supported. "
+                "Please use bytes input or XML fragments without declaration."
+            )
+            if str(ex) == msg:
+                raw_bytes = html_str.encode("utf-8", errors="ignore")
+                doc = lxml_html.fromstring(raw_bytes)
+                cleaned = self._cleaner.clean_html(doc)
+                rendered = lxml_html.tostring(cleaned, encoding="utf-8").decode("utf-8")
+                return minify_html.minify(rendered, **self.minify_kwargs) if self.minify else rendered
+            raise
+
+    def _strip_by_attribute_stop_words(self, *, soup: BeautifulSoup) -> None:
+        """Remove small, likely non-content elements by attribute tokens.
+
+        Scans leaf-like descendants under ``<body>`` and collects elements whose
+        ``id``, ``class``, ``role``, or ``item_type`` values contain any of the
+        configured ``attr_stop_words`` tokens (case-insensitive), then decomposes
+        them. Mirrors the baseline leaf-ness and concatenation behavior.
+
+        Parameters
+        ----------
+        soup : BeautifulSoup
+            Parsed document to prune in place.
+        """
+        body = soup.find("body") or soup
+        to_decompose: list[Tag] = []
+        for el in body.descendants:
+            if not isinstance(el, Tag):
+                continue
+            attrs = el.attrs if isinstance(el.attrs, dict) else {}
+            if not attrs:
+                continue
+            # Only prune simple leaf-ish nodes to avoid huge deletes unintentionally
+            if sum(1 for _ in el.descendants) > 1:
+                continue
+            for name in ("id", "class", "role", "item_type"):
+                val = attrs.get(name)
+                if val is None:
+                    continue
+                if isinstance(val, (list, tuple)):
+                    # Match baseline behavior: concatenate tokens without separator
+                    val_str = "".join(map(str, val))
+                else:
+                    val_str = str(val)
+                if any(sw in val_str.lower() for sw in self.attr_stop_words):
+                    to_decompose.append(el)
+                    break
+        for el in to_decompose:
+            el.decompose()
+
+    def _prune_header_footer(self, soup: BeautifulSoup) -> None:
+        """Prune likely navigational clutter inside header and footer.
+
+        Removes common list-like elements and links inside ``<header>``/``<footer>``
+        when the corresponding toggles are enabled.
+        """
+        header = soup.find("header")
+        footer = soup.find("footer")
+        if header and self.remove_header_lists:
+            self._decompose_tags(header, {"a", "img", "ol", "ul", "li"})
+        if footer and self.remove_footer_lists:
+            self._decompose_tags(footer, {"a", "img", "ol", "ul", "li"})
+
+    def _prune_body(self, soup: BeautifulSoup) -> None:
+        body = soup.find("body") or soup
+        always_remove = {
+            "input", "textarea", "button", "select", "option", "optgroup", "datalist",
+            "label", "fieldset", "legend", "output", "meter", "dialog", "form",
+            "search", "progress", "svg", "canvas", "use", "nav", "object", "noscript",
+        }
+        to_decompose: list[Tag] = []
+        for el in body.descendants:
+            if not isinstance(el, Tag):
+                continue
+            if not isinstance(el.name, str):
+                continue
+            if el.name in self.non_text_to_keep:
+                continue
+            if el.name in always_remove:
+                to_decompose.append(el)
+        for el in to_decompose:
+            el.decompose()
+
+    def _drop_empty_leaf_nodes(self, soup: BeautifulSoup) -> None:
+        """Iteratively remove empty leaves using the baseline's strict leaf check.
+
+        Walks leaf nodes (no descendants) and removes those with no text content,
+        excluding tags explicitly whitelisted in ``non_text_to_keep``.
+        """
+        body = soup.find("body") or soup
+        while True:
+            to_decompose: list[Tag] = []
+            for el in body.descendants:
+                if not isinstance(el, Tag):
+                    continue
+                if not isinstance(el.name, str):
+                    continue
+                if el.name in self.non_text_to_keep:
+                    continue
+                # Baseline leaf check: element must have zero descendants at all
+                if len(list(el.descendants)) != 0:
+                    continue
+                # Remove if no text once stripped
+                if (el.get_text() or "").strip():
+                    continue
+                to_decompose.append(el)
+            if not to_decompose:
+                break
+            for el in to_decompose:
+                el.decompose()
+
+    @staticmethod
+    def _decompose_tags(root: Tag, names: set[str]) -> None:
+        for el in list(root.descendants):
+            if isinstance(el, Tag) and isinstance(el.name, str) and el.name in names:
+                el.decompose()

smol_html-0.1.0.dist-info/METADATA

@@ -0,0 +1,127 @@
+Metadata-Version: 2.4
+Name: smol-html
+Version: 0.1.0
+Summary: Small, dependable HTML cleaner/minifier with sensible defaults
+Project-URL: Homepage, https://github.com/NosibleAI/smol-html
+Project-URL: Repository, https://github.com/NosibleAI/smol-html
+Project-URL: Issues, https://github.com/NosibleAI/smol-html/issues
+Author-email: Gareth Warburton <garethw738@gmail.com>, Stuart Reid <stuart@nosible.com>, Matthew Dicks <matthew@nosible.com>, Richard Taylor <richard@nosible.com>
+License: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: beautifulsoup4>=4.13.5
+Requires-Dist: lxml[html-clean]>=6.0.1
+Requires-Dist: minify-html>=0.16.4
+Description-Content-Type: text/markdown
+
+# smol-html
+
+Small, dependable HTML cleaner/minifier with sensible defaults.
+
+## Installation
+
+- pip: `pip install smol-html`
+- uv: `uv pip install smol-html`
+
+## Quick Start
+
+Clean an HTML string (or page contents):
+
+```python
+from smol_html import SmolHtmlCleaner
+
+html = """
+<html>
+  <head><title> Example </title></head>
+  <body>
+    <div>  Hello <span> world </span> </div>
+  </body>
+</html>
+"""
+
+# All constructor arguments are keyword-only and optional.
+cleaner = SmolHtmlCleaner()
+cleaned = cleaner.clean(raw_html=html)
+
+print(cleaned)
+```
+
+## Customization
+
+`SmolHtmlCleaner` exposes keyword-only parameters with practical defaults. You can:
+- Pass overrides to the constructor, or
+- Adjust attributes on the instance after creation.
+
+```python
+from smol_html import SmolHtmlCleaner
+
+cleaner = SmolHtmlCleaner()
+cleaner.attr_stop_words.add("advert")  # e.g., add a custom stop word
+```
+
+## Usage Examples
+
+Minimal:
+
+```python
+from smol_html import SmolHtmlCleaner
+
+cleaner = SmolHtmlCleaner()
+out = cleaner.clean(raw_html="<p>Hi <!-- note --> <a href='x'>link</a></p>")
+```
+
+Customize a few options:
+
+```python
+from smol_html import SmolHtmlCleaner
+
+cleaner = SmolHtmlCleaner(
+    attr_stop_words={"nav", "advert"},
+    remove_header_lists=False,
+    minify=True,
+)
+
+out = cleaner.clean(raw_html="<p>Hi</p>")
+```
+
+## Parameter Reference
+
+The most useful parameters, what they do, and when to change them:
+
+| Parameter | Type | Default | What it does | When to change |
+|---|---|---|---|---|
+| `non_text_to_keep` | `set[str]` | media/meta/table/`br` tags | Whitelist of empty/non-text tags to preserve (e.g., images, figures, tables, line breaks). | If important non-text elements are being removed or you want to keep/drop more empty tags. |
+| `attr_stop_words` | `set[str]` | common UI/navigation tokens | Tokens matched against `id`/`class`/`role`/`item_type` on small elements; matches are removed as likely non-content. | Add tokens like `advert`, `hero`, `menu` to aggressively drop UI chrome, or remove tokens if content is lost. |
+| `remove_header_lists` | `bool` | `True` | Removes links/lists/images within `<header>` to reduce nav clutter. | Set `False` if your header contains meaningful content you want to keep. |
+| `remove_footer_lists` | `bool` | `True` | Removes links/lists/images within `<footer>` to reduce boilerplate. | Set `False` for content-heavy footers you need. |
+| `minify` | `bool` | `True` | Minifies output HTML using `minify_html`. | Set `False` for readability or debugging; use `--pretty` in the CLI. |
+| `minify_kwargs` | `dict` | `{}` | Extra options passed to `minify_html.minify`. | Tune minification behavior (e.g., whitespace, comments) without changing cleaning. |
+| `meta` | `bool` | `False` | lxml Cleaner option: remove `<meta>` content when `True`. | Usually leave `False`; enable only for strict sanitation. |
+| `page_structure` | `bool` | `False` | lxml Cleaner option: remove page-structure tags (e.g., `<head>`, `<body>`) when `True`. | Rarely needed; keep `False` to preserve structure. |
+| `links` | `bool` | `True` | lxml Cleaner option: sanitize/clean links. | Leave `True` unless you need raw anchors untouched. |
+| `scripts` | `bool` | `False` | lxml Cleaner option: remove `<script>` tags when `True`. | Keep `False` to preserve scripts; usually safe to remove via `javascript=True` anyway. |
+| `javascript` | `bool` | `True` | lxml Cleaner option: remove JS and event handlers. | Set `False` only if you truly need inline JS (not recommended). |
+| `comments` | `bool` | `True` | lxml Cleaner option: remove HTML comments. | Set `False` to retain comments for debugging. |
+| `style` | `bool` | `True` | lxml Cleaner option: remove CSS and style attributes. | Set `False` to keep inline styles/CSS. |
+| `processing_instructions` | `bool` | `True` | lxml Cleaner option: remove processing instructions. | Rarely change; keep for safety. |
+| `embedded` | `bool` | `True` | lxml Cleaner option: remove embedded content (e.g., `<embed>`, `<object>`). | Set `False` to keep embedded media. |
+| `frames` | `bool` | `True` | lxml Cleaner option: remove frames/iframes. | Set `False` if iframes contain needed content. |
+| `forms` | `bool` | `True` | lxml Cleaner option: remove form elements. | Set `False` if you need to keep forms/inputs. |
+| `annoying_tags` | `bool` | `True` | lxml Cleaner option: remove tags considered "annoying" by lxml (e.g., `<blink>`, `<marquee>`). | Rarely change. |
+| `kill_tags` | `set[str] | None` | `None` | Additional explicit tags to remove entirely. | Add site-specific or custom tags to drop. |
+| `remove_unknown_tags` | `bool` | `True` | lxml Cleaner option: drop unknown/invalid tags. | Set `False` if you rely on custom elements. |
+| `safe_attrs_only` | `bool` | `True` | Only allow attributes listed in `safe_attrs`. | Set `False` if you need to keep arbitrary attributes. |
+| `safe_attrs` | `set[str]` | curated set | Allowed HTML attributes when `safe_attrs_only=True`. | Extend to keep additional attributes you trust. |

smol_html-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+smol_html/__init__.py,sha256=ea_KigbtSifgJYhoYvaco0CszvdBEX08BuT5qKILY7k,106
+smol_html/core.py,sha256=oJ7YAjocTVouCdmnxZYmIvx_30G5427uQn-nF8Q-rng,12095
+smol_html-0.1.0.dist-info/METADATA,sha256=CWySJr1U5V34e9i-x9BtQM64sKZ0ZbCOQI-5HYBZMr4,6390
+smol_html-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+smol_html-0.1.0.dist-info/licenses/LICENSE,sha256=88yg3BujRGq8MYlWhbrzB2YMNWJaXnBck3c7l23labs,1089
+smol_html-0.1.0.dist-info/RECORD,,

smol_html-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

smol_html-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Nosible Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.