bindery-cli 0.27.0__py3-none-any.whl

bindery/transforms.py

@@ -0,0 +1,579 @@
+"""Deterministic, well-formedness-only repair transforms for (X)HTML/XML text.
+
+Every transform is a pure function `str -> (str, int)` returning the rewritten text
+and how many fixes it made. None of them change document semantics: they only make
+already-intended markup well-formed (self-close void elements, turn undeclared named
+entities into numeric character references, escape stray ampersands, strip junk before
+the XML prolog, drop a duplicated root xmlns). Anything deeper than that is out of
+scope and is left for the epubcheck gate to reject. See spec.md.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterable
+from functools import wraps
+from html.entities import html5, name2codepoint
+
+Transform = Callable[[str], tuple[str, int]]
+
+VOID = "area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr"
+XML_PREDEFINED = {"amp", "lt", "gt", "quot", "apos"}
+
+# Match an open void-element tag. The lookahead after the name is essential: without
+# it `<col` matches inside `<colgroup>` and we self-close a non-void element, orphaning
+# its end-tag (the bug that introduced fatals on real books). A plain `\b` is not
+# enough either: `-`, `:`, and `.` are valid XML name characters but not \w, so `\b`
+# still matched `<col` inside a custom `<col-group>`. Attributes are matched
+# quote-aware so a `>` inside an attribute value does not end the tag early. Group 3
+# captures an existing trailing slash so already-self-closed tags are left untouched.
+_VOID_RE = re.compile(
+    rf"""<({VOID})(?=[\s/>])((?:"[^"]*"|'[^']*'|[^>])*?)\s*(/?)>""",
+    re.IGNORECASE | re.DOTALL,
+)
+_VOID_END_RE = re.compile(rf"""</(?:{VOID})\s*>""", re.IGNORECASE)
+_NAMED_ENTITY_RE = re.compile(r"&([a-zA-Z][a-zA-Z0-9]*);")
+_BARE_AMP_RE = re.compile(r"&(?![a-zA-Z][a-zA-Z0-9]*;|#[0-9]+;|#[xX][0-9a-fA-F]+;)")
+# A start tag (quote-aware) and an attribute within it, for invalid-attribute stripping.
+_START_TAG_RE = re.compile(r"""<[a-zA-Z][\w:.-]*(?:"[^"]*"|'[^']*'|[^>])*>""")
+# The unquoted-value branch stops at whitespace or the end of the tag, and the `/?>`
+# lookahead is what keeps it off the tag's own self-closing slash: a plain `[^\s>]+`
+# swallowed it, so dropping the attribute in `<img 31=x/>` left `<img>` and turned a
+# well-formed tag into an unclosed one. A `/` mid-value (a URL) is still consumed,
+# because only a `/` immediately before `>` can be the tag's.
+_ATTR_RE = re.compile(
+    r"""(\s+)([^\s=/>]+)(\s*=\s*)("[^"]*"|'[^']*'|[^\s>]*?(?=\s|/?>))"""
+)
+_XMLNS_DECL_RE = re.compile(r"xmlns:([A-Za-z_][\w.-]*)\s*=")
+_HTML_TAG_RE = re.compile(r"<html\b[^>]*>", re.IGNORECASE)
+_XMLNS_ATTR_RE = re.compile(r'\s+xmlns="[^"]*"')
+_EPUB_PREFIX_ATTR_RE = re.compile(
+    r"""(?:\bepub:)?prefix\s*=\s*(?:"([^"]*)"|'([^']*)')"""
+)
+_EPUB_PREFIX_VAL_RE = re.compile(r"(?:^|\s)([A-Za-z_][\w.-]*)\s*:\s*\S+")
+
+# CDATA sections and comments hold literal text: a bare `&`, an entity name, or a
+# `<br>` inside them is already legal XML, and rewriting it would change the content
+# (e.g. `&` in CDATA-wrapped CSS/JS renders as `&`; escaped, it renders as `&amp;`).
+# Splitting on this regex yields alternating outside/protected segments, so the
+# markup transforms run only on the even (outside) indices.
+_PROTECTED_RE = re.compile(r"(<!\[CDATA\[.*?\]\]>|<!--.*?-->)", re.DOTALL)
+
+
+def _outside_protected(fn: Transform) -> Transform:
+    """Wrap a transform so it never touches CDATA sections or comments."""
+
+    @wraps(fn)
+    def wrapped(s: str) -> tuple[str, int]:
+        if "<!" not in s:  # fast path: nothing to protect
+            return fn(s)
+        parts = _PROTECTED_RE.split(s)
+        total = 0
+        for i in range(0, len(parts), 2):
+            parts[i], n = fn(parts[i])
+            total += n
+        return "".join(parts), total
+
+    return wrapped
+
+
+@_outside_protected
+def self_close_void(s: str) -> tuple[str, int]:
+    """Self-close void elements that were left open (`<br>` -> `<br/>`).
+
+    Already-self-closed tags are returned unchanged and not counted, so the fix is
+    idempotent and reports only real changes.
+    """
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        if m.group(0).endswith("/>"):
+            return m.group(0)
+        count += 1
+        return f"<{m.group(1)}{m.group(2)}/>"
+
+    s = _VOID_RE.sub(repl, s)
+
+    # Strip any remaining end tags for void elements (e.g. </br>) which would
+    # otherwise cause fatal XML parse errors since we self-closed their start tags.
+    s, end_count = _VOID_END_RE.subn("", s)
+    count += end_count
+
+    return s, count
+
+
+def strip_invalid_attributes(s: str) -> tuple[str, int]:
+    """Remove attributes that make the XML unparseable: a name starting with a digit
+    (e.g. a mangled `31=""`), or a namespaced name whose prefix is not declared anywhere
+    in the document (e.g. Office VML `v:shapes` with no `xmlns:v`).
+
+    A well-formed document has no such attributes by definition, so this is a no-op on
+    good files and only touches already-malformed ones. The fix is surgical: only the
+    offending attribute is dropped, everything else is preserved byte-for-byte.
+    """
+    # The declared-prefix set is computed over the whole document (over-collecting
+    # from comments only makes the fix more conservative), but tags are rewritten
+    # only outside CDATA/comment spans.
+    declared = set(_XMLNS_DECL_RE.findall(s)) | {"xml", "xmlns"}
+    for m in _EPUB_PREFIX_ATTR_RE.finditer(s):
+        val = m.group(1) or m.group(2) or ""
+        declared.update(_EPUB_PREFIX_VAL_RE.findall(val))
+    count = 0
+
+    def fix_tag(tag: re.Match) -> str:
+        def drop(attr: re.Match) -> str:
+            nonlocal count
+            name = attr.group(2)
+            prefix = name.split(":", 1)[0] if ":" in name else None
+            if name[0].isdigit() or (prefix is not None and prefix not in declared):
+                count += 1
+                return ""
+            return attr.group(0)
+
+        return _ATTR_RE.sub(drop, tag.group(0))
+
+    parts = _PROTECTED_RE.split(s)
+    for i in range(0, len(parts), 2):
+        parts[i] = _START_TAG_RE.sub(fix_tag, parts[i])
+    return "".join(parts), count
+
+
+def _entity_refs(name: str) -> str | None:
+    """Numeric character reference(s) for an HTML entity name, or None if unknown.
+
+    Most entities are one codepoint; the handful that expand to several (`&fjlig;`,
+    `&NotEqualTilde;`, ...) become one numeric reference per codepoint, which renders
+    identically.
+    """
+    if name in name2codepoint:
+        return f"&#{name2codepoint[name]};"
+    ch = html5.get(name + ";") or html5.get(name)
+    if not ch:
+        return None
+    return "".join(f"&#{ord(c)};" for c in ch)
+
+
+@_outside_protected
+def fix_named_entities(s: str) -> tuple[str, int]:
+    """Replace undeclared HTML named entities with numeric refs (`&nbsp;` -> `&#160;`).
+
+    XML only predefines five entity names; everything else (`&nbsp;`, `&deg;`,
+    `&eacute;`, ...) is a fatal "entity not declared" unless turned into a numeric
+    reference, which every XML parser understands.
+    """
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        name = m.group(1)
+        if name in XML_PREDEFINED:
+            return m.group(0)
+        refs = _entity_refs(name)
+        if refs is None:
+            return m.group(0)
+        count += 1
+        return refs
+
+    return _NAMED_ENTITY_RE.sub(repl, s), count
+
+
+# A DOCTYPE internal subset can declare custom entities (<!DOCTYPE x [<!ENTITY ...>]>),
+# making an "unknown" name legitimate; the escape skips such documents wholesale.
+_INTERNAL_SUBSET_RE = re.compile(r"<!DOCTYPE[^>\[]*\[", re.IGNORECASE)
+
+
+@_outside_protected
+def _escape_unknown_part(s: str) -> tuple[str, int]:
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        name = m.group(1)
+        if name in XML_PREDEFINED or _entity_refs(name) is not None:
+            return m.group(0)
+        count += 1
+        return "&amp;" + m.group(0)[1:]
+
+    return _NAMED_ENTITY_RE.sub(repl, s), count
+
+
+def escape_unknown_entities(s: str) -> tuple[str, int]:
+    """Escape entity references whose name is unknown (`&foo;` -> `&amp;foo;`).
+
+    An undeclared entity outside the HTML5 table stays a fatal "entity not declared";
+    escaping it renders exactly as browsers already render an unknown entity (the
+    literal text `&foo;`). Conditionally semantics-preserving: identical rendering
+    EXCEPT against a document whose DOCTYPE internal subset declares the entity, so
+    any document carrying an internal subset is skipped wholesale. Opt-in
+    (--escape-unknown-entities), never a core transform.
+    """
+    if _INTERNAL_SUBSET_RE.search(s):
+        return s, 0
+    return _escape_unknown_part(s)
+
+
+# An <img> start tag, quote-aware like _VOID_RE; group 1 is the attribute block and
+# group 2 an existing self-closing slash, so the tag is rebuilt without reordering.
+_IMG_TAG_RE = re.compile(
+    r"""<img(?=[\s/>])((?:"[^"]*"|'[^']*'|[^>])*?)\s*(/?)>""",
+    re.IGNORECASE | re.DOTALL,
+)
+# Whitespace before `alt` keeps data-alt/xml:alt-style names from matching; a quoted
+# value containing ` alt=` can false-positive, which only makes the fix skip that tag.
+_ALT_PRESENT_RE = re.compile(r"""(?:^|\s)alt\s*=""", re.IGNORECASE)
+
+
+@_outside_protected
+def add_img_alt(s: str) -> tuple[str, int]:
+    """Add `alt=""` to an <img> that has no alt attribute (the RSC-005 "missing
+    required attribute" error).
+
+    Rendering is unchanged (empty alt draws nothing), but this is the one transform
+    that ADDS markup the author never wrote, and `alt=""` asserts "decorative" to a
+    screen reader where a missing alt did not. Hence opt-in (--add-img-alt), never a
+    core transform. Idempotent: a tag that already carries alt is left untouched.
+    """
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        attrs, slash = m.group(1), m.group(2)
+        if _ALT_PRESENT_RE.search(attrs):
+            return m.group(0)
+        count += 1
+        return f'<img{attrs} alt=""{slash}>'
+
+    return _IMG_TAG_RE.sub(repl, s), count
+
+
+@_outside_protected
+def escape_bare_amp(s: str) -> tuple[str, int]:
+    """Escape a `&` that does not begin a valid entity/character reference."""
+    return _BARE_AMP_RE.subn("&amp;", s)
+
+
+def strip_prolog_junk(s: str) -> tuple[str, int]:
+    """Remove a BOM or stray bytes before the first `<` ("content not allowed in prolog").
+
+    Leading whitespace is junk only when an XML declaration follows it (a declaration
+    must be the very first thing in the document). Before a DOCTYPE or the root element
+    it is legal prolog whitespace and is left alone: counting it as a fix marked the
+    document changed, which forced repair_epub's decode("utf-8", "replace") round-trip
+    on a file that had nothing wrong with it.
+    """
+    stripped = s.lstrip(" \t\r\n")
+    i = stripped.find("<")
+    if i > 0:
+        stripped = stripped[i:]
+    if stripped == s:
+        return s, 0
+    removed = s[: len(s) - len(stripped)]
+    if not removed.strip(" \t\r\n") and not stripped.startswith("<?xml"):
+        return s, 0
+    return stripped, 1
+
+
+def drop_duplicate_xmlns(s: str) -> tuple[str, int]:
+    """Keep only the first `xmlns="..."` on the root <html> element."""
+    m = _HTML_TAG_RE.search(s)
+    if not m:
+        return s, 0
+    tag = m.group(0)
+    seen = False
+    count = 0
+
+    def repl(mm: re.Match) -> str:
+        nonlocal seen, count
+        if seen:
+            count += 1
+            return ""
+        seen = True
+        return mm.group(0)
+
+    new_tag = _XMLNS_ATTR_RE.sub(repl, tag)
+    if count:
+        s = s[: m.start()] + new_tag + s[m.start() + len(tag) :]
+    return s, count
+
+
+# Transforms applied to full (X)HTML content documents, in order. Prolog and root-tag
+# fixes first, then ampersand/entity normalization, then void self-closing.
+@_outside_protected
+def fix_ncx_playorder(s: str) -> tuple[str, int]:
+    count = 0
+    playorder = 1
+
+    def repl(m: re.Match) -> str:
+        nonlocal count, playorder
+        val = m.group(2)
+        if val != str(playorder):
+            count += 1
+        res = f'{m.group(1)}"{playorder}"'
+        playorder += 1
+        return res
+
+    s, n = re.subn(
+        r'(playOrder\s*=\s*)["\']([^"\']+)["\']', repl, s, flags=re.IGNORECASE
+    )
+    return s, count
+
+
+@_outside_protected
+def unwrap_block_in_inline(s: str) -> tuple[str, int]:
+    """Unwrap a <span> that illegally wraps a block element (`<span><div>...</div>
+    </span>` -> the div alone).
+
+    Opt-in (--unwrap-block-in-inline): it restructures nesting, not just
+    well-formedness tokens, so it is never part of the core pass. Inner text is
+    always preserved.
+    """
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return m.group(1)
+
+    s, n = re.subn(
+        r"<span[^>]*>\s*(<(div|p|blockquote)[^>]*>.*?</\2>)\s*</span>",
+        repl,
+        s,
+        flags=re.IGNORECASE | re.DOTALL,
+    )
+    return s, n
+
+
+@_outside_protected
+def strip_invalid_value(s: str) -> tuple[str, int]:
+    """Strip misplaced `value="..."` attributes from non-form elements.
+
+    Opt-in (--strip-invalid-value): deleting attributes goes beyond making markup
+    parseable, so it is never part of the core pass.
+    """
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        tag = m.group(1)
+        before = m.group(2) or ""
+        after = m.group(4) or ""
+        return f"<{tag} {before}{after}>"
+
+    s, n = re.subn(
+        r'<(div|span|p|a|img|h[1-6]|ul|li|meta|table|tr|td|th)(\s+[^>]*\b)?value\s*=\s*(["\'][^"\']*["\'])([^>]*)>',
+        repl,
+        s,
+        flags=re.IGNORECASE,
+    )
+    return s, n
+
+
+# The tags --unwrap-illegal-tags removes wherever they appear. A tag name that any
+# stylesheet references as an element selector is skipped for that book (see
+# css_protected_tags): unwrapping styled elements would silently destroy formatting,
+# which no lossy convenience is worth.
+ILLEGAL_TAGS = ("st", "sentence", "o", "w", "pagebreak")
+
+_CSS_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL)
+_STYLE_BLOCK_RE = re.compile(r"<style\b[^>]*>(.*?)</style>", re.IGNORECASE | re.DOTALL)
+
+
+def css_protected_tags(
+    *sheets: str, tags: tuple[str, ...] = ILLEGAL_TAGS
+) -> frozenset[str]:
+    """Element-selector-referenced tag names in the given CSS text(s),
+    from `tags` (default: the illegal-tag set).
+
+    Selector-aware, so `.st { ... }` (a class) or `#w { ... }` (an id) does NOT
+    protect anything while `st { ... }` or `p st, x > w { ... }` does. Comments are
+    stripped first; matching is case-insensitive and returns lowercase names. This
+    ports scripts/find_css_illegal_tags.py into the library so the transform can
+    enforce its own precondition instead of trusting callers to scan.
+    """
+    found: set[str] = set()
+    for css in sheets:
+        stripped = _CSS_COMMENT_RE.sub("", css)
+        # Candidate selector lists are whatever precedes a '{'. Scanning this way
+        # (rather than splitting on '}') keeps nested at-rules working: in
+        # `@media print { o > sentence {} }`, `o > sentence` precedes the INNER '{'
+        # and is found, where a naive outer split would lose it. Declarations never
+        # precede a '{'.
+        for m in re.finditer(r"([^{}]*)\{", stripped):
+            selectors = m.group(1)
+            for tag in tags:
+                # The trailing boundary includes . and #: `pagebreak.new:after`
+                # styles pagebreak ELEMENTS, so it must protect the name, while a
+                # LEADING . or # stays unprotected (`div.st` styles a class).
+                if re.search(
+                    rf"(^|[\s,>+~]){tag}([\s,>+~:\[.#]|$)", selectors, re.IGNORECASE
+                ):
+                    found.add(tag)
+    return frozenset(found)
+
+
+def style_block_tags(doc: str, tags: tuple[str, ...] = ILLEGAL_TAGS) -> frozenset[str]:
+    """css_protected_tags over every inline `<style>` block in a content document."""
+    return css_protected_tags(*_STYLE_BLOCK_RE.findall(doc), tags=tags)
+
+
+def unwrap_illegal_tags(
+    s: str, protected_tags: frozenset[str] = frozenset()
+) -> tuple[str, int]:
+    """Remove illegal/deprecated tags (`<st>`, `<sentence>`, `<o>`, `<w>`,
+    `<pagebreak>`) outright, keeping their inner text.
+
+    Opt-in (`--unwrap-illegal-tags`) because deleting elements is more than
+    well-formedness repair. Names in `protected_tags` (the union of every
+    stylesheet's element selectors, see css_protected_tags/style_block_tags) are
+    left untouched: if a book styles one of these tags, unwrapping it would change
+    how the text renders, so preservation wins.
+    """
+    count = 0
+    for tag in ILLEGAL_TAGS:
+        if tag in protected_tags:
+            continue
+        s, n1 = re.subn(rf"<{tag}\b[^>]*>", "", s, flags=re.IGNORECASE)
+        s, n2 = re.subn(rf"</{tag}\s*>", "", s, flags=re.IGNORECASE)
+        count += n1 + n2
+    return s, count
+
+
+@_outside_protected
+def fix_empty_body(s: str) -> tuple[str, int]:
+    """Append `&nbsp;` to a strictly empty `<body></body>` ("body incomplete").
+
+    Opt-in (--fix-empty-body): this ADDS visible content the author never wrote,
+    the same reason --add-img-alt is opt-in; it is never part of the core pass.
+    """
+    count = 0
+
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return m.group(1) + "&nbsp;" + m.group(2)
+
+    s, n = re.subn(r"(<body[^>]*>\s*)(</body>)", repl, s, flags=re.IGNORECASE)
+    return s, n
+
+
+@_outside_protected
+def fix_missing_title(s: str) -> tuple[str, int]:
+    """Inject `<title>Unknown</title>` when a document has no usable title.
+
+    Opt-in (--fix-missing-title): it fabricates content the author never wrote,
+    so it is never part of the core pass. An existing non-empty <title> anywhere
+    in the document means no-op.
+    """
+    count = 0
+    if re.search(r"<title\b[^>]*>.*?</title>", s, re.IGNORECASE | re.DOTALL):
+        return s, 0
+
+    # Check for empty self-closing title
+    def repl(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return "<title>Unknown</title>"
+
+    s, n = re.subn(r"<title\b[^>]*>\s*</title>", repl, s, flags=re.IGNORECASE)
+    if n > 0:
+        return s, n
+
+    # No title tag at all, inject into head
+    def repl_head(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return m.group(1) + "\n<title>Unknown</title>"
+
+    s, n = re.subn(r"(<head[^>]*>)", repl_head, s, flags=re.IGNORECASE)
+    return s, n
+
+
+@_outside_protected
+def fix_id_colons(s: str) -> tuple[str, int]:
+    """Translate illegal colons in `id="X:Y"` and matching `#X:Y` fragments to `_`.
+
+    Opt-in (--fix-id-colons): it rewrites every id-bearing attribute and internal
+    fragment reference, so it stays out of the core pass despite being
+    rendering-neutral. Word boundaries keep external URLs intact.
+    """
+    count = 0
+
+    def repl_id(m: re.Match) -> str:
+        nonlocal count
+        count += 1
+        return m.group(1) + m.group(2).replace(":", "_") + m.group(3)
+
+    s, n1 = re.subn(
+        r'\b(id\s*=\s*["\'])([^"\']+)(["\'])', repl_id, s, flags=re.IGNORECASE
+    )
+
+    def repl_href(m: re.Match) -> str:
+        nonlocal count
+        if ":" in m.group(3):
+            count += 1
+            return (
+                m.group(1)
+                + m.group(2)
+                + "#"
+                + m.group(3).replace(":", "_")
+                + m.group(4)
+            )
+        return m.group(0)
+
+    s, n2 = re.subn(
+        r'\b(href\s*=\s*["\'])([^"#]*?)#([^"\']+)(["\'])',
+        repl_href,
+        s,
+        flags=re.IGNORECASE,
+    )
+    return s, count
+
+
+# The always-on core: exactly the five semantics-preserving well-formedness fixes
+# the spec names, nothing else. Everything that adds markup, deletes attributes, or
+# removes/restructures elements lives behind an explicit CLI flag (threaded through
+# repair_epub), so the default pass can never change more than parsing requires.
+# See spec.md "Transforms" and the per-flag sections.
+HTML_TRANSFORMS = (
+    strip_prolog_junk,
+    drop_duplicate_xmlns,
+    escape_bare_amp,
+    fix_named_entities,
+    self_close_void,
+)
+
+# A lighter set for XML sidecars (NCX): no HTML-specific element rewriting.
+XML_TRANSFORMS = (
+    strip_prolog_junk,
+    escape_bare_amp,
+    fix_named_entities,
+    fix_ncx_playorder,
+)
+
+
+def apply_transforms(
+    s: str, transforms: Iterable[Transform]
+) -> tuple[str, dict[str, int]]:
+    """Run a pipeline of transforms, returning the result and per-transform counts."""
+    counts: dict[str, int] = {}
+    for fn in transforms:
+        s, n = fn(s)
+        if n:
+            counts[fn.__name__] = counts.get(fn.__name__, 0) + n
+    return s, counts
+
+
+# Matches leaked closing tags without angle brackets (e.g. /div&gt;) OR common formatting tags with escaped angle brackets.
+_BROKEN_TAGS_RE = re.compile(
+    r"(?<!<)(?<!&lt;)/[a-zA-Z]+&gt;|&lt;/(?:p|div|span|h[1-6]|i|em|b|strong)&gt;",
+    re.IGNORECASE,
+)
+
+
+def strip_broken_tags(text: str) -> tuple[str, int]:
+    """Strip leaked HTML closing tags that render as raw text in readers.
+    This lossily removes the leaked text fragment and should be gated behind `validate.no_worse`.
+    """
+    return _BROKEN_TAGS_RE.subn("", text)