justhtml 0.24.0__py3-none-any.whl → 0.38.0__py3-none-any.whl

justhtml/__init__.py

@@ -1,22 +1,64 @@
 from .parser import JustHTML, StrictModeError
-from .sanitize import DEFAULT_DOCUMENT_POLICY, DEFAULT_POLICY, SanitizationPolicy, UrlRule, sanitize
+from .sanitize import (
+    CSS_PRESET_TEXT,
+    DEFAULT_DOCUMENT_POLICY,
+    DEFAULT_POLICY,
+    SanitizationPolicy,
+    UnsafeHtmlError,
+    UrlPolicy,
+    UrlProxy,
+    UrlRule,
+)
 from .selector import SelectorError, matches, query
 from .serialize import to_html, to_test_format
 from .stream import stream
 from .tokens import ParseError
+from .transforms import (
+    CollapseWhitespace,
+    Decide,
+    Drop,
+    Edit,
+    EditAttrs,
+    EditDocument,
+    Empty,
+    Linkify,
+    PruneEmpty,
+    RewriteAttrs,
+    Sanitize,
+    SetAttrs,
+    Stage,
+    Unwrap,
+)
 
 __all__ = [
+    "CSS_PRESET_TEXT",
     "DEFAULT_DOCUMENT_POLICY",
     "DEFAULT_POLICY",
+    "CollapseWhitespace",
+    "Decide",
+    "Drop",
+    "Edit",
+    "EditAttrs",
+    "EditDocument",
+    "Empty",
     "JustHTML",
+    "Linkify",
     "ParseError",
+    "PruneEmpty",
+    "RewriteAttrs",
     "SanitizationPolicy",
+    "Sanitize",
     "SelectorError",
+    "SetAttrs",
+    "Stage",
     "StrictModeError",
+    "UnsafeHtmlError",
+    "Unwrap",
+    "UrlPolicy",
+    "UrlProxy",
     "UrlRule",
     "matches",
     "query",
-    "sanitize",
     "stream",
     "to_html",
     "to_test_format",

justhtml/__main__.py

@@ -32,7 +32,7 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
             "  curl -s https://example.com | justhtml -\n"
             "  justhtml page.html --selector 'main p' --format text\n"
             "  justhtml page.html --selector 'a' --format html\n"
-            "  justhtml page.html --selector 'article' --format markdown\n"
+            "  justhtml page.html --selector 'article' --allow-tags article --format markdown\n"
             "\n"
             "If you don't have the 'justhtml' command available, use:\n"
             "  python -m justhtml ...\n"
@@ -62,6 +62,14 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
         action="store_true",
         help="Disable sanitization (trusted input only)",
     )
+
+    parser.add_argument(
+        "--allow-tags",
+        help=(
+            "Safe mode: allow these additional tags during sanitization (comma-separated). "
+            "Example: --allow-tags article,section"
+        ),
+    )
     parser.add_argument(
         "--first",
         action="store_true",
@@ -123,7 +131,36 @@ def main() -> None:
     args = _parse_args(sys.argv[1:])
     html = _read_html(args.path)
     fragment_context = FragmentContext("div") if args.fragment else None
-    doc = JustHTML(html, fragment_context=fragment_context)
+    safe = not args.unsafe
+
+    policy = None
+    if safe and args.allow_tags:
+        from .sanitize import DEFAULT_DOCUMENT_POLICY, DEFAULT_POLICY, SanitizationPolicy  # noqa: PLC0415
+
+        extra_tags: set[str] = set()
+        for part in str(args.allow_tags).replace(" ", ",").split(","):
+            tag = part.strip().lower()
+            if tag:
+                extra_tags.add(tag)
+
+        base = DEFAULT_POLICY if fragment_context is not None else DEFAULT_DOCUMENT_POLICY
+        allowed = set(base.allowed_tags)
+        allowed.update(extra_tags)
+        policy = SanitizationPolicy(
+            allowed_tags=allowed,
+            allowed_attributes=base.allowed_attributes,
+            url_policy=base.url_policy,
+            drop_comments=base.drop_comments,
+            drop_doctype=base.drop_doctype,
+            drop_foreign_namespaces=base.drop_foreign_namespaces,
+            drop_content_tags=base.drop_content_tags,
+            allowed_css_properties=base.allowed_css_properties,
+            force_link_rel=base.force_link_rel,
+            unsafe_handling=base.unsafe_handling,
+            disallowed_tag_handling=base.disallowed_tag_handling,
+        )
+
+    doc = JustHTML(html, fragment_context=fragment_context, safe=safe, policy=policy)
 
     try:
         nodes = doc.query(args.selector) if args.selector else [doc.root]
@@ -138,9 +175,8 @@ def main() -> None:
         nodes = [nodes[0]]
 
     def write_output(out: TextIO) -> None:
-        safe = not args.unsafe
         if args.format == "html":
-            outputs = [node.to_html(safe=safe) for node in nodes]
+            outputs = [node.to_html() for node in nodes]
             out.write("\n".join(outputs))
             out.write("\n")
             return
@@ -149,19 +185,19 @@ def main() -> None:
             # Keep these branches explicit so coverage will highlight untested CLI options.
             if args.separator == " ":
                 if args.strip:
-                    outputs = [node.to_text(strip=True, safe=safe) for node in nodes]
+                    outputs = [node.to_text(strip=True) for node in nodes]
                 else:
-                    outputs = [node.to_text(strip=False, safe=safe) for node in nodes]
+                    outputs = [node.to_text(strip=False) for node in nodes]
             else:
                 if args.strip:
-                    outputs = [node.to_text(separator=args.separator, strip=True, safe=safe) for node in nodes]
+                    outputs = [node.to_text(separator=args.separator, strip=True) for node in nodes]
                 else:
-                    outputs = [node.to_text(separator=args.separator, strip=False, safe=safe) for node in nodes]
+                    outputs = [node.to_text(separator=args.separator, strip=False) for node in nodes]
             out.write("\n".join(outputs))
             out.write("\n")
             return
 
-        outputs = [node.to_markdown(safe=safe) for node in nodes]
+        outputs = [node.to_markdown() for node in nodes]
         out.write("\n\n".join(outputs))
         out.write("\n")
 

justhtml/constants.py

@@ -184,6 +184,18 @@ HTML4_PUBLIC_PREFIXES = (
 
 HEADING_ELEMENTS = {"h1", "h2", "h3", "h4", "h5", "h6"}
 
+# Elements where pretty-printing and whitespace-collapsing transforms should
+# preserve text node whitespace.
+WHITESPACE_PRESERVING_ELEMENTS: Final[frozenset[str]] = frozenset(
+    {
+        "code",
+        "pre",
+        "script",
+        "style",
+        "textarea",
+    }
+)
+
 FORMATTING_ELEMENTS = {
     "a",
     "b",

justhtml/errors.py

@@ -1,7 +1,8 @@
-"""Centralized error message definitions and helpers for HTML parsing errors.
+"""Centralized error message definitions and helpers for JustHTML errors.
 
-This module provides human-readable error messages for all parse error codes
-emitted by both the tokenizer and tree builder during HTML parsing.
+This module provides human-readable error messages for parse error codes
+emitted by the tokenizer and tree builder during HTML parsing, plus selected
+security findings emitted by the sanitizer.
 """
 
 from __future__ import annotations
@@ -143,6 +144,10 @@ def generate_error_message(code: str, tag_name: str | None = None) -> str:
         "unexpected-start-tag-in-select": f"Unexpected <{tag_name}> start tag in <select>",
         "unexpected-end-tag-in-select": f"Unexpected </{tag_name}> end tag in <select>",
         "unexpected-select-in-select": "Unexpected nested <select> in <select>",
+        # ================================================================
+        # SECURITY ERRORS
+        # ================================================================
+        "unsafe-html": "Unsafe HTML detected by sanitization policy",
     }
 
     # Return message or fall back to the code itself if not found

justhtml/linkify.py

@@ -0,0 +1,438 @@
+"""Text linkification scanner.
+
+This module finds URL/email-like substrings in plain text.
+
+It is intentionally HTML-agnostic: in JustHTML it is applied to DOM text nodes,
+not to raw HTML strings.
+
+The behavior is driven by vendored compliance fixtures from the upstream
+`linkify-it` project (MIT licensed). See `tests/linkify-it/README.md`.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class LinkMatch:
+    start: int
+    end: int
+    text: str
+    href: str
+    kind: str  # "url" | "email"
+
+
+DEFAULT_TLDS: Final[frozenset[str]] = frozenset(
+    {
+        # Keep this aligned with linkify-it's default list.
+        # See: https://github.com/markdown-it/linkify-it/blob/master/index.mjs
+        "biz",
+        "com",
+        "edu",
+        "gov",
+        "net",
+        "org",
+        "pro",
+        "web",
+        "xxx",
+        "aero",
+        "asia",
+        "coop",
+        "info",
+        "museum",
+        "name",
+        "shop",
+        "рф",
+    }
+)
+
+
+# A pragmatic Unicode-aware domain label pattern.
+#
+# Use `\w` for Unicode letters/digits (and underscore), and reject underscores
+# during validation. This is intentionally stricter than allowing all non-ASCII
+# codepoints, and matches the fixture behavior around delimiter punctuation.
+_LABEL_RE: Final[str] = (
+    r"[0-9A-Za-z\w\u2600-\u27bf]"
+    r"(?:[0-9A-Za-z\w\u2600-\u27bf-]{0,61}[0-9A-Za-z\w\u2600-\u27bf])?"
+)
+
+# A fast-ish candidate matcher. We do real validation after we find a candidate.
+_CANDIDATE_PATTERN: Final[str] = "".join(
+    [
+        r"(?i)([^0-9A-Za-z_])",  # left boundary (avoid matching after underscore)
+        r"(",  # candidate group
+        r"(?:https?|ftp)://[^\s<>\uFF5C]+",  # absolute URL
+        r"|mailto:[^\s<>\uFF5C]+",  # mailto
+        r"|//[^\s<>\uFF5C]+",  # protocol-relative
+        r"|(?:www\.)[^\s<>\uFF5C]+",  # www.
+        rf"|[0-9A-Za-z.!#$%&'*+/=?^_`{{|}}~\-\"]+@(?:{_LABEL_RE}\.)+{_LABEL_RE}",  # email
+        r"|(?:\d{1,3}\.){3}\d{1,3}(?:/[^\s<>\uFF5C]*)?",  # IPv4
+        rf"|(?:{_LABEL_RE}\.)+{_LABEL_RE}(?:/[^\s<>\uFF5C]*)?",  # fuzzy domain/path
+        r")",
+    ]
+)
+
+_CANDIDATE_RE: Final[re.Pattern[str]] = re.compile(_CANDIDATE_PATTERN, re.UNICODE)
+
+_TRAILING_PUNCT: Final[str] = ".,;:!?"
+
+# RE pattern for 2-character TLDs, copied from linkify-it (MIT licensed).
+_CC_TLD_RE: Final[re.Pattern[str]] = re.compile(
+    r"^(?:a[cdefgilmnoqrstuwxz]|b[abdefghijmnorstvwyz]|c[acdfghiklmnoruvwxyz]|d[ejkmoz]|e[cegrstu]|f[ijkmor]|g[abdefghilmnpqrstuwy]|h[kmnrtu]|i[delmnoqrst]|j[emop]|k[eghimnprwyz]|l[abcikrstuvy]|m[acdeghklmnopqrstuvwxyz]|n[acefgilopruz]|om|p[aefghklmnrstwy]|qa|r[eosuw]|s[abcdeghijklmnortuvxyz]|t[cdfghjklmnortvwz]|u[agksyz]|v[aceginu]|w[fs]|y[et]|z[amw])$",
+    re.IGNORECASE,
+)
+
+
+def _is_valid_tld(tld: str, *, extra_tlds: frozenset[str]) -> bool:
+    t = (tld or "").lower()
+    if not t:
+        return False
+    # Only valid 2-letter ccTLDs (avoid false positives like `.js`).
+    if len(t) == 2 and _CC_TLD_RE.match(t) is not None:
+        return True
+    # Any punycode root.
+    if t.startswith("xn--"):
+        return True
+    return t in DEFAULT_TLDS or t in extra_tlds
+
+
+def _split_domain_for_tld(host: str) -> tuple[str, str] | None:
+    # Return (domain_without_tld, tld).
+    h = (host or "").strip().strip(".")
+    if not h:
+        return None
+    if h.lower() == "localhost":
+        return ("localhost", "")
+    if "." not in h:
+        return None
+    base, tld = h.rsplit(".", 1)
+    return (base, tld)
+
+
+@dataclass(frozen=True, slots=True)
+class LinkifyConfig:
+    fuzzy_ip: bool = False
+    extra_tlds: frozenset[str] = frozenset()
+
+    @staticmethod
+    def with_extra_tlds(extra_tlds: list[str] | tuple[str, ...] | set[str] | frozenset[str]) -> LinkifyConfig:
+        return LinkifyConfig(extra_tlds=frozenset(str(t).lower() for t in extra_tlds))
+
+
+def _is_valid_ipv4(host: str) -> bool:
+    parts = host.split(".")
+    if len(parts) != 4:
+        return False
+    for p in parts:
+        if not p or len(p) > 3:
+            return False
+        if not p.isdigit():
+            return False
+        v = int(p)
+        if v < 0 or v > 255:
+            return False
+    return True
+
+
+def _punycode_host(host: str) -> str:
+    # Safety default: normalize Unicode domains to punycode for href.
+    try:
+        return host.encode("idna").decode("ascii")
+    except UnicodeError:
+        return host
+
+
+def _split_host_and_rest(raw: str) -> tuple[str, str]:
+    # raw is after an optional scheme prefix (or for fuzzy domains, the whole).
+    # Extract host[:port] and the rest (path/query/fragment).
+    for i, ch in enumerate(raw):
+        if ch in "/?#":
+            return raw[:i], raw[i:]
+    return raw, ""
+
+
+def _strip_wrapping(raw: str) -> tuple[str, int, int]:
+    # Trim common wrappers like <...> or quotes, but report how many chars were removed
+    # from start/end so we can compute accurate offsets.
+    start_trim = 0
+    end_trim = 0
+
+    if raw and raw[0] in "<\"'([{" and raw[-1] in ">\"')]}":
+        # Angle brackets are common for autolinks.
+        # Quotes/brackets: we strip them only if they wrap the candidate.
+        raw = raw[1:-1]
+        start_trim = 1
+        end_trim = 1
+
+    return raw, start_trim, end_trim
+
+
+def _trim_trailing(candidate: str) -> str:
+    # Remove trailing punctuation and unbalanced closing brackets.
+    if not candidate:
+        return candidate
+
+    # First strip sentence punctuation.
+    while candidate and candidate[-1] in _TRAILING_PUNCT:
+        candidate = candidate[:-1]
+
+    # Then strip quoting terminators when unbalanced (treat quotes as wrappers).
+    while candidate and candidate[-1] in "\"'":
+        q = candidate[-1]
+        if candidate.count(q) % 2 == 1:
+            candidate = candidate[:-1]
+            continue
+        break
+
+    # Then strip unmatched closing brackets.
+    # We treat ) ] } > as potentially closable.
+    pairs = {")": "(", "]": "[", "}": "{", ">": "<"}
+    while candidate and candidate[-1] in pairs:
+        close = candidate[-1]
+        open_ch = pairs[close]
+        if candidate.count(close) > candidate.count(open_ch):
+            candidate = candidate[:-1]
+            continue
+        break
+
+    return candidate
+
+
+def _href_for(text: str) -> tuple[str, str]:
+    lower = text.lower()
+
+    if lower.startswith("mailto:"):
+        return text, "email"
+
+    if "@" in text and not lower.startswith(("http://", "https://", "ftp://", "//", "www.")):
+        return f"mailto:{text}", "email"
+
+    if lower.startswith(("http://", "https://", "ftp://", "//")):
+        return text, "url"
+
+    # www. and fuzzy domains default to http://
+    return f"http://{text}", "url"
+
+
+def _punycode_href(href: str) -> str:
+    # Convert the host portion to punycode (IDNA), keeping the rest intact.
+    lower = href.lower()
+    prefix = ""
+    rest = href
+
+    if lower.startswith("mailto:"):
+        return href
+
+    if lower.startswith("http://"):
+        prefix = href[:7]
+        rest = href[7:]
+    elif lower.startswith("https://"):
+        prefix = href[:8]
+        rest = href[8:]
+    elif lower.startswith("ftp://"):
+        prefix = href[:6]
+        rest = href[6:]
+    elif lower.startswith("//"):
+        prefix = href[:2]
+        rest = href[2:]
+    else:
+        # Shouldn't happen; fuzzy hrefs are normalized before calling.
+        prefix = ""
+        rest = href
+
+    hostport, tail = _split_host_and_rest(rest)
+
+    # Handle userinfo (user:pass@host)
+    userinfo = ""
+    hostport2 = hostport
+    if "@" in hostport:
+        userinfo, hostport2 = hostport.rsplit("@", 1)
+
+    host = hostport2
+    port = ""
+    if hostport2.startswith("["):
+        # IPv6-ish, don't punycode.
+        return href
+    if ":" in hostport2:
+        host, port = hostport2.split(":", 1)
+
+    host_pc = _punycode_host(host)
+    rebuilt = host_pc
+    if port:
+        rebuilt = f"{rebuilt}:{port}"
+    if userinfo:
+        rebuilt = f"{userinfo}@{rebuilt}"
+
+    return f"{prefix}{rebuilt}{tail}"
+
+
+def find_links(text: str) -> list[LinkMatch]:
+    return find_links_with_config(text, LinkifyConfig())
+
+
+def find_links_with_config(text: str, config: LinkifyConfig) -> list[LinkMatch]:
+    if not text:
+        return []
+
+    # Mirror linkify-it behavior: always scan with a leading boundary character.
+    scan_text = "\n" + text
+
+    out: list[LinkMatch] = []
+
+    for m in _CANDIDATE_RE.finditer(scan_text):
+        raw = m.group(2)
+
+        # Compute absolute offsets (exclude the boundary prefix char).
+        start = m.start(2) - 1
+        end = m.end(2) - 1
+
+        stripped, s_trim, e_trim = _strip_wrapping(raw)
+        start += s_trim
+        end -= e_trim
+
+        cand = _trim_trailing(stripped)
+        if not cand:
+            continue
+
+        # Markdown-style termination: `(...URL...)[...]` should stop at the `)`.
+        lower = cand.lower()
+        if lower.startswith(("http://", "https://", "ftp://")) and ")[" in cand:
+            cand = cand.split(")[", 1)[0]
+            cand = _trim_trailing(cand)
+            if not cand:
+                continue
+
+        # Treat leading quotes as wrappers/delimiters, not part of the URL/email.
+        if cand and cand[0] in "\"'" and 0 <= start < len(text) and text[start] == cand[0]:
+            cand = cand[1:]
+            start += 1
+            if not cand:
+                continue
+
+        # Adjust end after trimming.
+        end = start + len(cand)
+
+        lower = cand.lower()
+
+        # If this looks like a fuzzy domain that starts immediately after ://,
+        # treat it as part of a broken/disabled schema (e.g. _http://example.com, hppt://example.com).
+        if not lower.startswith(("http://", "https://", "ftp://", "mailto:", "//", "www.")) and "@" not in cand:
+            if start >= 3 and text[start - 3 : start] == "://":
+                continue
+            if start > 0 and text[start - 1] in "/:@":
+                continue
+
+        # Validate fuzzy IP option.
+        if (
+            cand
+            and cand[0].isdigit()
+            and "." in cand
+            and not lower.startswith(("http://", "https://", "ftp://", "//"))
+        ):
+            host, _ = _split_host_and_rest(cand)
+            if host.replace(".", "").isdigit() and _is_valid_ipv4(host):
+                if not config.fuzzy_ip:
+                    continue
+
+        # Validate // URLs: allow localhost or dotted domains, but not single-level.
+        if lower.startswith("//"):
+            # Protect against matching the // inside :// or ///.
+            if start > 0 and text[start - 1] in ":/":
+                continue
+            after = cand[2:]
+            hostport, _ = _split_host_and_rest(after)
+            if not hostport:
+                continue
+            if hostport.startswith("["):
+                continue
+            host_only = hostport
+            if "@" in host_only:
+                host_only = host_only.rsplit("@", 1)[1]
+            if ":" in host_only:
+                host_only = host_only.split(":", 1)[0]
+            if host_only.lower() != "localhost" and "." not in host_only:
+                continue
+
+            if "_" in host_only:
+                continue
+
+        # Validate fuzzy domains and emails with TLD allowlist.
+        is_scheme = lower.startswith(("http://", "https://", "ftp://", "mailto:"))
+        is_www = lower.startswith("www.")
+        is_proto_rel = lower.startswith("//")
+
+        if not is_scheme and not is_proto_rel and not is_www and "@" not in cand:
+            host, _ = _split_host_and_rest(cand)
+            if "_" in host:
+                continue
+
+            # IPv4 candidates don't use the TLD allowlist.
+            if "." in host and host.replace(".", "").isdigit() and _is_valid_ipv4(host):
+                pass
+            else:
+                parts = _split_domain_for_tld(host)
+                if parts is None:
+                    continue
+                _base, tld = parts
+                if not _is_valid_tld(tld, extra_tlds=config.extra_tlds):
+                    continue
+
+        if (
+            "@" in cand
+            and not lower.startswith(("http://", "https://", "ftp://", "//"))
+            and not lower.startswith("mailto:")
+        ):
+            # Fuzzy email requires a valid TLD.
+            local, domain = cand.rsplit("@", 1)
+            _ = local
+            host, _tail = _split_host_and_rest(domain)
+            if "_" in host:
+                continue
+            parts = _split_domain_for_tld(host)
+            if parts is None:
+                continue
+            _base, tld = parts
+            if not _is_valid_tld(tld, extra_tlds=config.extra_tlds):
+                continue
+
+        # Validate basic URL host/port if scheme-based.
+        if lower.startswith(("http://", "https://", "ftp://")):
+            after = cand.split("://", 1)[1]
+            hostport, _ = _split_host_and_rest(after)
+            if not hostport:
+                continue
+            if "@" in hostport:
+                hostport = hostport.rsplit("@", 1)[1]
+            host = hostport
+            if ":" in hostport and not hostport.startswith("["):
+                host, port = hostport.split(":", 1)
+                if port and (not port.isdigit() or int(port) > 65535):
+                    continue
+            if not host or host.startswith(("-", ".")) or host.endswith(("-", ".")) or ".." in host:
+                continue
+            if "_" in host:
+                continue
+            if "." in host and host.replace(".", "").isdigit() and not _is_valid_ipv4(host):
+                continue
+
+        href, kind = _href_for(cand)
+        href = _punycode_href(href)
+
+        out.append(LinkMatch(start=start, end=end, text=cand, href=href, kind=kind))
+
+    # Avoid overlapping matches by keeping first-longest.
+    if not out:
+        return out
+    out.sort(key=lambda x: (x.start, -(x.end - x.start)))
+    filtered: list[LinkMatch] = []
+    last_end = -1
+    for lm in out:
+        if lm.start < last_end:
+            continue
+        filtered.append(lm)
+        last_end = lm.end
+    return filtered