bindery-cli 0.27.0__py3-none-any.whl

bindery/validate.py

@@ -0,0 +1,254 @@
+"""epubcheck wrapper and the acceptance gate.
+
+epubcheck is the external oracle. A repair is only worth keeping if it strictly
+reduces problems and never introduces new ones. If epubcheck is not installed, the
+gate degrades safely: validation is skipped and callers must decide whether to trust
+the repair without it (the CLI requires --no-validate to do so).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+_SUMMARY_RE = re.compile(
+    r"Messages:\s*(\d+)\s+fatals?\s*/\s*(\d+)\s+errors?\s*/\s*(\d+)\s+warnings?"
+)
+
+
+@dataclass(frozen=True)
+class CheckResult:
+    fatals: int
+    errors: int
+    warnings: int
+
+    def __str__(self) -> str:
+        return f"{self.fatals}f/{self.errors}e/{self.warnings}w"
+
+
+def epubcheck_available() -> bool:
+    return shutil.which("epubcheck") is not None
+
+
+def _english_locale_env() -> dict[str, str]:
+    """The subprocess env with the JVM pinned to English (roadmap 5.2).
+
+    epubcheck localizes its human-readable summary, which broke the regex
+    fallback on non-English locales. Appended rather than assigned so a
+    user's existing JAVA_TOOL_OPTIONS (heap flags etc.) survive.
+    """
+    env = dict(os.environ)
+    opts = env.get("JAVA_TOOL_OPTIONS", "")
+    env["JAVA_TOOL_OPTIONS"] = f"{opts} -Duser.language=en -Duser.country=US".strip()
+    return env
+
+
+def _counts_from_json(stdout: str) -> CheckResult | None:
+    """Parse counts from epubcheck's locale-independent `--json -` output.
+
+    epubcheck 5.x puts the totals under "checker" (nFatal/nError/nWarning,
+    verified against 5.3.0). Anything unexpected returns None so the caller
+    can fall back to the summary-line regex.
+    """
+    try:
+        data = json.loads(stdout)
+    except json.JSONDecodeError:
+        return None
+    checker = data.get("checker") if isinstance(data, dict) else None
+    if not isinstance(checker, dict):
+        return None
+    try:
+        return CheckResult(
+            int(checker["nFatal"]), int(checker["nError"]), int(checker["nWarning"])
+        )
+    except KeyError, TypeError, ValueError:
+        return None
+
+
+_DAEMON_JAVA = """
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.Scanner;
+import com.adobe.epubcheck.api.EpubCheck;
+import com.adobe.epubcheck.reporting.CheckingReport;
+
+public class FastDaemon {
+    public static void main(String[] args) throws Exception {
+        Scanner scanner = new Scanner(System.in);
+        while (scanner.hasNextLine()) {
+            String path = scanner.nextLine();
+            if (path.trim().isEmpty()) continue;
+            File epub = new File(path);
+            if (!epub.exists()) {
+                System.out.println("-1,-1,-1");
+                continue;
+            }
+            try {
+                PrintWriter out = new PrintWriter(new java.io.OutputStream() {
+                    public void write(int b) {}
+                });
+                CheckingReport report = new CheckingReport(out, epub.getName());
+                EpubCheck check = new EpubCheck(epub, report);
+                check.doValidate();
+                System.out.println(report.getFatalErrorCount() + "," + report.getErrorCount() + "," + report.getWarningCount());
+            } catch (Exception e) {
+                System.out.println("-1,-1,-1");
+            }
+        }
+    }
+}
+"""
+
+
+class _EpubcheckDaemon:
+    def __init__(self):
+        self._proc = None
+        self._lock = __import__("threading").Lock()
+
+    def _start(self):
+        epubcheck_bin = shutil.which("epubcheck")
+        if not epubcheck_bin:
+            return False
+
+        try:
+            with open(epubcheck_bin) as f:
+                script = f.read()
+            m = re.search(
+                r"java\s+(?:-[^ ]+\s+)*-jar\s+[\"'\\]*([^\s\"'\\]+\.jar)", script
+            )
+            if not m:
+                return False
+            jar_path = os.path.expandvars(m.group(1))
+            if not os.path.exists(jar_path):
+                return False
+
+            self.workdir = __import__("tempfile").mkdtemp(prefix="bindery-daemon-")
+            java_file = os.path.join(self.workdir, "FastDaemon.java")
+            with open(java_file, "w") as f:
+                f.write(_DAEMON_JAVA)
+
+            subprocess.run(
+                ["javac", "-cp", jar_path, java_file],
+                check=True,
+                capture_output=True,
+            )
+
+            self._proc = subprocess.Popen(
+                ["java", "-cp", f".:{jar_path}", "FastDaemon"],
+                cwd=self.workdir,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL,
+                text=True,
+                bufsize=1,
+            )
+            __import__("atexit").register(self.stop)
+            return True
+        except Exception:
+            return False
+
+    def check(self, path: Path) -> CheckResult | None:
+        with self._lock:
+            if self._proc is None:
+                if not self._start():
+                    return None
+            try:
+                self._proc.stdin.write(str(path.resolve()) + "\n")
+                self._proc.stdin.flush()
+                res = self._proc.stdout.readline().strip()
+                if not res or res == "-1,-1,-1":
+                    return None
+                f, e, w = map(int, res.split(","))
+                return CheckResult(f, e, w)
+            except Exception:
+                return None
+
+    def stop(self):
+        if self._proc:
+            if self._proc.stdin:
+                try:
+                    self._proc.stdin.close()
+                except OSError:
+                    pass
+            self._proc.terminate()
+            self._proc.wait(timeout=2)
+            self._proc = None
+        if hasattr(self, "workdir") and os.path.exists(self.workdir):
+            shutil.rmtree(self.workdir, ignore_errors=True)
+
+
+_daemon = _EpubcheckDaemon()
+
+
+def run_epubcheck(path: Path, timeout: int = 300) -> CheckResult | None:
+    """Run epubcheck and return parsed counts, or None if it could not be parsed.
+
+    Counts come from `--json -` (locale-independent) first; the English summary-line
+    regex stays as the fallback for epubchecks too old for `--json`, kept meaningful
+    by the JVM locale pin in the env. The persistent daemon answers first when it is
+    available; None anywhere means the caller falls back to (or reports) failure.
+    """
+    res = _daemon.check(path)
+    if res is not None:
+        return res
+    try:
+        out = subprocess.run(
+            ["epubcheck", str(path), "--json", "-"],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            env=_english_locale_env(),
+            check=False,
+        )
+    except FileNotFoundError, subprocess.TimeoutExpired:
+        return None
+    result = _counts_from_json(out.stdout)
+    if result is not None:
+        return result
+    m = _SUMMARY_RE.search(out.stdout + out.stderr)
+    if not m:
+        return None
+    return CheckResult(int(m.group(1)), int(m.group(2)), int(m.group(3)))
+
+
+def gate(before: CheckResult, after: CheckResult) -> str:
+    """Classify a repair: 'accept', 'partial', 'reject', or 'noop'.
+
+    The metric depends on whether the book started with fatals, because a fatal parse
+    error halts epubcheck on that file and hides every downstream schema error. So:
+
+    - Started WITH fatals: success is fewer fatals. A rising error count is just those
+      latent errors becoming visible once the file parses (the book now opens), not a
+      regression. 'accept' if all fatals cleared, 'partial' if merely reduced.
+    - Started with NO fatals (pure error/NCX-001 cleanup): nothing was masking errors,
+      so an error increase is a real regression. Require a strict error decrease.
+
+    Introducing net-new fatals is always a 'reject'.
+    """
+    if after.fatals > before.fatals:
+        return "reject"
+    if before.fatals > 0:
+        if after.fatals == 0:
+            return "accept"
+        return "partial" if after.fatals < before.fatals else "noop"
+    if after.errors > before.errors:
+        return "reject"
+    return "accept" if after.errors < before.errors else "noop"
+
+
+def no_worse(before: CheckResult, after: CheckResult) -> bool:
+    """The acceptance bar for a lossy content repair (page-number stripping), whose
+    benefit epubcheck cannot see. Unlike `gate`, it does not demand a measured
+    improvement; it only forbids a regression: no net-new fatals, and no new errors
+    unless fatals were already masking them. Mirrors oceanstrip's 'no more fatals or
+    errors than the original' bar."""
+    if after.fatals > before.fatals:
+        return False
+    if before.fatals == 0 and after.errors > before.errors:
+        return False
+    return True

bindery/watermark.py

@@ -0,0 +1,244 @@
+"""Locate and remove producer/conversion watermarks from (X)HTML documents.
+
+A watermark is a run of markup whose visible text is a known producer stamp.
+Producers inject it two ways, and oceanstrip handles both with the same
+balanced-element removal so a well-formed document stays well-formed:
+
+- **Anchored** (a link): an <a> whose start tag (href) carries a known signature.
+  OceanofPDF appends `<a href="...oceanofpdf.com">OceanofPDF.com</a>`; ABC Amber
+  wraps `<b>Generated by ABC Amber LIT Conv<a href="...processtext...">erter,
+  ...</a></b>` (the word "Converter" straddles the tag boundary). We find the <a>,
+  walk up to the OUTERMOST enclosing wrapper whose entire visible text is the
+  watermark, and delete that whole balanced element. If no such clean wrapper
+  exists (the link sits inline in real prose), only the <a> is removed.
+
+- **Anchorless** (plain text): the stamp is literal text with the URL not linked,
+  e.g. `<p>ABC Amber LIT Converter http://www.processtext.com/abclit.html</p>` or a
+  `<div type="FOOTER"><p><span>...</span></p></div>`. There is no <a> to anchor on,
+  so we locate the stamp by a text signature and walk up to the outermost wrapper
+  whose entire visible text is the watermark (allowing the stamp to repeat), then
+  delete it. Real prose that merely mentions the URL is safe: its wrapper's text is
+  not *only* the watermark, so nothing is removed.
+
+A flat regex cannot do either safely: it closes on the wrong tag and corrupts the
+XML, or orphans the half of a phrase that sits outside the <a>. New producers are
+one entry in WATERMARKS; the passes below are otherwise signature-agnostic.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class Watermark:
+    """A known producer watermark.
+
+    - ``href_sig``: lowercase substring expected in an anchored watermark's <a>
+      start tag (its href), or None if this producer never links the stamp.
+    - ``text_sig``: a lowercase substring of the stamp's *visible text* used to
+      locate an anchorless occurrence cheaply (e.g. "processtext.com/abclit").
+    - ``phrase``: a regex (case-insensitive) matching one occurrence of the stamp's
+      full visible text. A wrapper is removed only when its entire normalized text
+      is one or more repetitions of this phrase, which is what keeps real prose that
+      merely sits near the stamp untouched.
+
+    ``anchor`` and ``pure`` are the compiled patterns, built in __post_init__.
+    """
+
+    name: str
+    href_sig: str | None
+    text_sig: str
+    phrase: str
+    anchor: re.Pattern[str] | None = field(default=None, compare=False)
+    pure: re.Pattern[str] = field(default=None, compare=False)  # type: ignore[assignment]
+
+    def __post_init__(self) -> None:
+        anchor = (
+            re.compile(
+                rf"<a\b[^>]*?{re.escape(self.href_sig)}[^>]*?>.*?</a>",
+                re.IGNORECASE | re.DOTALL,
+            )
+            if self.href_sig
+            else None
+        )
+        object.__setattr__(self, "anchor", anchor)
+        object.__setattr__(
+            self, "pure", re.compile(rf"(?:{self.phrase}\s*)+", re.IGNORECASE)
+        )
+
+
+# The known watermarks. Append here to support a new producer; nothing else changes.
+WATERMARKS: tuple[Watermark, ...] = (
+    Watermark(
+        name="OceanofPDF",
+        href_sig="oceanofpdf",
+        text_sig="oceanofpdf.com",
+        phrase=r"oceanofpdf\.com",
+    ),
+    Watermark(
+        name="ABC Amber LIT Converter",
+        href_sig="processtext",
+        text_sig="processtext.com/abclit",
+        phrase=r"(?:generated by )?abc amber lit converter,?\s*"
+        r"https?://(?:www\.)?processtext\.com/abclit\.html",
+    ),
+)
+
+# Any tag (open / close / self-closing), used for the balanced-element walk.
+TAG = re.compile(r"<(/?)([a-zA-Z][\w:-]*)\b[^>]*?(/?)>", re.DOTALL)
+
+# HTML void elements never have a closing tag; ignore them in balance tracking.
+VOID = {
+    "area",
+    "base",
+    "br",
+    "col",
+    "embed",
+    "hr",
+    "img",
+    "input",
+    "link",
+    "meta",
+    "param",
+    "source",
+    "track",
+    "wbr",
+}
+
+# Removable watermark wrappers. Never body/html/section, so a page whose sole *text*
+# is the watermark (e.g. an image-only title page) is not deleted wholesale. The
+# inline elements (b/i/em/strong) are here because ABC Amber wraps its stamp in a
+# <b>; the "visible text is only the watermark" guard below keeps the wider set
+# safe: a real heading/paragraph/bold phrase won't have text that is a watermark.
+WRAPPERS = {
+    "div",
+    "p",
+    "span",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "b",
+    "i",
+    "em",
+    "strong",
+}
+
+# If a candidate wrapper contains any of these it holds real content. don't remove.
+MEDIA = re.compile(
+    r"<(?:img|image|svg|video|audio|picture|iframe|table|object)\b", re.IGNORECASE
+)
+
+
+def _norm(element: str) -> str:
+    """Visible text of an element fragment: tags stripped, nbsp and whitespace runs
+    collapsed to single spaces, stripped, lowercased."""
+    text = re.sub(r"<[^>]+>", "", element).replace("\xa0", " ")
+    return re.sub(r"\s+", " ", text).strip().lower()
+
+
+def _matching_close(html: str, name: str, open_end: int) -> tuple[int, int] | None:
+    """Span of the close tag matching a `name` element already opened just before
+    `open_end` (which must point past that opening tag's '>')."""
+    depth = 1
+    pat = re.compile(
+        rf"<(/?)({re.escape(name)})\b[^>]*?(/?)>", re.IGNORECASE | re.DOTALL
+    )
+    for m in pat.finditer(html, open_end):
+        closing, _, selfclose = m.group(1), m.group(2), m.group(3)
+        if selfclose:
+            continue
+        if closing:
+            depth -= 1
+            if depth == 0:
+                return m.start(), m.end()
+        else:
+            depth += 1
+    return None
+
+
+def _ancestors(html: str, a_start: int, a_end: int):
+    """Yield (open_start, close_end) for wrapper elements enclosing [a_start, a_end],
+    outermost first."""
+    stack: list[tuple[str, int, int]] = []
+    for m in TAG.finditer(html, 0, a_start):
+        closing, name, selfclose = m.group(1), m.group(2).lower(), m.group(3)
+        if name in VOID or selfclose:
+            continue
+        if closing:
+            for i in range(len(stack) - 1, -1, -1):
+                if stack[i][0] == name:
+                    del stack[i:]
+                    break
+        else:
+            stack.append((name, m.start(), m.end()))
+
+    for name, ostart, oend in stack:
+        if name not in WRAPPERS:
+            continue
+        close = _matching_close(html, name, oend)
+        if close and close[0] >= a_end:
+            yield ostart, close[1]
+
+
+def _pure_wrapper_span(
+    html: str, start: int, end: int, wm: Watermark
+) -> tuple[int, int] | None:
+    """The outermost wrapper enclosing [start, end] whose entire visible text is the
+    watermark (one or more repetitions) and which holds no media; else None."""
+    for ostart, cend in _ancestors(html, start, end):  # outermost first
+        element = html[ostart:cend]
+        if MEDIA.search(element):
+            continue
+        if wm.pure.fullmatch(_norm(element)):
+            return ostart, cend
+    return None
+
+
+def _strip_anchored(html: str, wm: Watermark) -> tuple[str, int]:
+    """Remove anchored (linked) occurrences of `wm`. Deletes the outermost pure
+    wrapper around each watermark <a>, or the <a> itself when it sits inline."""
+    if wm.anchor is None:
+        return html, 0
+    count = 0
+    while (m := wm.anchor.search(html)) is not None:
+        span = _pure_wrapper_span(html, m.start(), m.end(), wm)
+        s, e = span if span else (m.start(), m.end())
+        html = html[:s] + html[e:]
+        count += 1
+    return html, count
+
+
+def _strip_text(html: str, wm: Watermark) -> tuple[str, int]:
+    """Remove anchorless (plain-text) occurrences of `wm`. Finds the stamp by its
+    text signature and deletes the outermost wrapper whose whole text is the
+    watermark; an occurrence with no such wrapper (the URL sits in real prose) is
+    left untouched."""
+    count = 0
+    pos = 0
+    while (i := html.lower().find(wm.text_sig, pos)) != -1:
+        span = _pure_wrapper_span(html, i, i + len(wm.text_sig), wm)
+        if span is None:
+            pos = i + len(wm.text_sig)  # not a pure watermark wrapper; skip it
+            continue
+        html = html[: span[0]] + html[span[1] :]
+        count += 1
+        pos = 0
+    return html, count
+
+
+def strip_watermark_html(html: str) -> tuple[str, int]:
+    """Remove every known watermark (anchored and anchorless) from one document.
+
+    Returns (cleaned_html, number_of_watermarks_removed).
+    """
+    total = 0
+    for wm in WATERMARKS:
+        html, n_anchor = _strip_anchored(html, wm)
+        html, n_text = _strip_text(html, wm)
+        total += n_anchor + n_text
+    return html, total