parseoff 0.2.1__py3-none-any.whl

parseoff/runner.py

@@ -0,0 +1,263 @@
+"""Find PDFs, run every selected parser in a worker subprocess, cache results, compute metrics."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from . import findings, metrics, textlayer
+from .adapters.custom import CustomSpec
+from .i18n import LocalizedError, t
+from .model import DocReport, ParserOutput, RunReport, Skipped, Thresholds
+from .registry import REGISTRY, installed_version, is_installed
+
+CACHE_SCHEMA = 1
+STDERR_LIMIT = 2000
+
+
+class UsageError(LocalizedError):
+    """Bad input from the user (exit code 2). The message says how to fix it.
+
+    ``str(error)`` is English. The command line shows ``error.text(lang)`` in the screen language.
+    """
+
+
+@dataclass(frozen=True)
+class ParserSpec:
+    name: str
+    custom: CustomSpec | None = None
+
+
+@dataclass
+class RunOptions:
+    input_path: Path
+    out_dir: Path
+    parsers: list[str] | None = None  # None = automatic selection
+    customs: list[CustomSpec] = field(default_factory=list)
+    max_pages: int | None = None
+    timeout: float = 120.0
+    fresh: bool = False
+    render: bool = True
+    thresholds: Thresholds = field(default_factory=Thresholds)
+    demo: bool = False
+    lang: str = "en"  # language of the progress lines only; results never depend on it
+
+
+# --- document discovery -------------------------------------------------------------------------
+
+
+def find_pdfs(path: Path) -> list[tuple[str, Path]]:
+    """Return (doc_id, file) pairs sorted by doc_id. doc_id is relative to the input folder."""
+    if not path.exists():
+        raise UsageError("usage.no_such", path=path)
+    if path.is_file():
+        if path.suffix.lower() != ".pdf":
+            raise UsageError("usage.not_pdf", path=path)
+        return [(path.name, path)]
+    found = sorted(
+        ((p.relative_to(path).as_posix(), p) for p in path.rglob("*") if p.is_file() and p.suffix.lower() == ".pdf"),
+        key=lambda pair: pair[0],
+    )
+    if not found:
+        raise UsageError("usage.no_pdfs", path=path)
+    return found
+
+
+def make_slugs(doc_ids: list[str]) -> list[str]:
+    """Filesystem- and URL-safe names derived from the relative path; duplicates get -2, -3 ..."""
+    slugs: list[str] = []
+    seen: dict[str, int] = {}
+    for doc_id in doc_ids:
+        base = re.sub(r"[^A-Za-z0-9._-]+", "_", re.sub(r"\.pdf$", "", doc_id, flags=re.I)).strip("._") or "doc"
+        seen[base] = seen.get(base, 0) + 1
+        slugs.append(base if seen[base] == 1 else f"{base}-{seen[base]}")
+    return slugs
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for block in iter(lambda: fh.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+# --- parser selection ---------------------------------------------------------------------------
+
+
+def select_parsers(
+    requested: list[str] | None, customs: list[CustomSpec]
+) -> tuple[list[ParserSpec], list[Skipped]]:
+    """Decide which parsers run. Missing optional parsers are skipped (with an install hint)."""
+    names = list(dict.fromkeys(requested)) if requested else [n for n, i in REGISTRY.items() if i.auto]
+    unknown = [n for n in names if n not in REGISTRY]
+    if unknown:
+        raise UsageError("usage.unknown_parsers", unknown=", ".join(unknown), known=", ".join(REGISTRY))
+    selected: list[ParserSpec] = []
+    skipped: list[Skipped] = []
+    for name in REGISTRY:  # registry order, so reports always list parsers in the same order
+        info = REGISTRY[name]
+        if name not in names:
+            if not requested and not info.auto:
+                skipped.append(Skipped(name, t("skip.opt_in", name=name), "skip.opt_in", {"name": name}))
+            continue
+        if is_installed(name):
+            selected.append(ParserSpec(name))
+        else:
+            skipped.append(Skipped(
+                name, t("skip.not_installed", hint=info.install_hint or t("skip.install_first")),
+                "skip.not_installed", {"name": name},
+            ))
+    selected += [ParserSpec(c.name, c) for c in customs]
+    return selected, skipped
+
+
+# --- running one parser -------------------------------------------------------------------------
+
+
+def _launch(cmd: list[str], timeout: float, env: dict[str, str]) -> tuple[int | None, bytes]:
+    """Run a worker. Returns (exit code or None on timeout, stderr bytes). Kills it on timeout."""
+    try:
+        done = subprocess.run(cmd, capture_output=True, timeout=timeout, env=env, check=False)
+        return done.returncode, done.stderr
+    except subprocess.TimeoutExpired as exc:
+        return None, exc.stderr or b""
+
+
+def _scrub(text: str, roots: dict[str, str]) -> str:
+    """Replace absolute paths with placeholders so reports can be shared and compared."""
+    for root in sorted(roots, key=len, reverse=True):
+        if root:
+            text = text.replace(root, roots[root])
+    return text
+
+
+def _cache_key(sha: str, spec: ParserSpec, version: str, pages: list[int]) -> dict:
+    return {"schema": CACHE_SCHEMA, "sha256": sha, "parser": spec.name, "version": version, "pages": pages}
+
+
+def run_parser(
+    spec: ParserSpec,
+    pdf: Path,
+    sha: str,
+    pages: list[int],
+    raw_dir: Path,
+    options: RunOptions,
+    roots: dict[str, str],
+) -> ParserOutput:
+    raw_dir.mkdir(parents=True, exist_ok=True)
+    final = raw_dir / f"{spec.name}.json"
+    version = "custom" if spec.custom else (installed_version(spec.name) or "")
+    key = _cache_key(sha, spec, version, pages)
+    # custom parsers are never cached: their code can change without any version change
+    cacheable = not spec.custom and bool(version)
+
+    if cacheable and not options.fresh and final.is_file():
+        try:
+            stored = json.loads(final.read_text(encoding="utf-8"))
+            if stored.get("key") == key and stored["output"]["status"] == "ok":
+                return ParserOutput(**{**stored["output"], "cached": True})
+        except (ValueError, KeyError, TypeError):
+            pass  # unreadable cache entry: just run the parser again
+
+    tmp = raw_dir / f"{spec.name}.running.json"
+    tmp.unlink(missing_ok=True)
+    cmd = [sys.executable, "-m", "parseoff.worker", "--pdf", str(pdf), "--pages", ",".join(map(str, pages)), "--out", str(tmp)]
+    cmd += ["--custom", spec.custom.spec] if spec.custom else ["--parser", spec.name]
+    env = {**os.environ, "PYTHONIOENCODING": "utf-8", "PARSEOFF_PATH": os.pathsep.join(p for p in [os.environ.get("PARSEOFF_PATH", ""), *sys.path] if p)}
+    code, raw_err = _launch(cmd, options.timeout, env)
+    stderr = _scrub(raw_err.decode("utf-8", "replace"), roots)[-STDERR_LIMIT:]
+
+    if code is None:
+        output = ParserOutput(
+            spec.name, "timeout", version, float(options.timeout), [],
+            t("err.timeout", seconds=options.timeout), stderr,
+            error_key="err.timeout", error_params={"seconds": options.timeout},
+        )
+    else:
+        try:
+            data = json.loads(tmp.read_text(encoding="utf-8"))
+            error = _scrub(str(data.get("error", "")), roots)
+            output = ParserOutput(
+                spec.name, "error" if error else "ok", data.get("version") or version,
+                float(data.get("seconds", 0.0)), list(data.get("pages", [])), error, stderr,
+            )
+        except (OSError, ValueError):
+            output = ParserOutput(
+                spec.name, "error", version, 0.0, [],
+                t("err.worker_exit", code=code), stderr,
+                error_key="err.worker_exit", error_params={"code": code},
+            )
+    tmp.unlink(missing_ok=True)
+    stored = {"key": key, "output": {k: v for k, v in output.__dict__.items() if k != "cached"}}
+    final.write_text(json.dumps(stored, ensure_ascii=True, indent=1, sort_keys=True), encoding="utf-8")
+    return output
+
+
+# --- the whole pipeline -------------------------------------------------------------------------
+
+
+def run_pipeline(options: RunOptions, progress: Callable[[str], None] = lambda _msg: None) -> RunReport:
+    """Run every selected parser over every PDF and return a report with metrics and findings."""
+    lang = options.lang
+    docs = find_pdfs(options.input_path)
+    specs, skipped = select_parsers(options.parsers, options.customs)
+    slugs = make_slugs([doc_id for doc_id, _ in docs])
+    root = options.input_path.resolve()
+    roots = {
+        str(root): "<input>", str(options.out_dir.resolve()): "<out>", os.getcwd(): "<cwd>",
+        str(Path.home()): "~", sys.prefix: "<python>", sys.base_prefix: "<python>",
+    }
+    img_dir = options.out_dir / "report" / "img"
+    reports: list[DocReport] = []
+
+    for (doc_id, pdf), slug in zip(docs, slugs):
+        doc = DocReport(doc_id=doc_id, slug=slug)
+        reports.append(doc)
+        try:
+            doc.sha256 = file_sha256(pdf)
+            doc.total_pages = textlayer.page_count(pdf)
+            if doc.total_pages == 0:
+                raise ValueError("the PDF has no pages")
+            last = min(doc.total_pages, options.max_pages or doc.total_pages)
+            doc.pages = list(range(1, last + 1))
+            layer = textlayer.layer_texts(pdf, doc.pages)
+        except Exception as exc:  # noqa: BLE001 - one unreadable file must not stop the run
+            doc.open_error = _scrub(f"{type(exc).__name__}: {exc}", roots)
+            doc.pages = []
+            progress(t("progress.cannot_open", lang, doc=doc_id, error=doc.open_error))
+            continue
+        doc.layer_chars = [len(t.strip()) for t in layer]
+        if options.render:
+            try:
+                textlayer.render_pages(pdf, doc.pages, img_dir, slug)
+                doc.rendered = True
+            except Exception as exc:  # noqa: BLE001 - images are a convenience, not required
+                progress(t("progress.images_skipped", lang, doc=doc_id, error=type(exc).__name__))
+        for spec in specs:
+            out = run_parser(spec, pdf, doc.sha256, doc.pages, options.out_dir / "raw" / slug, options, roots)
+            doc.outputs[spec.name] = out
+            note = t("progress.cached", lang) if out.cached else ""
+            progress(t("progress.parser", lang, doc=doc_id, parser=spec.name, status=out.status,
+                       seconds=out.seconds, cached=note))
+            if out.status != "ok" and out.error.strip():
+                # show the cause in the terminal too, not only in summary.md
+                # (our own messages are translated; text from a parser or library stays as it is)
+                cause = t(out.error_key, lang, **out.error_params) if out.error_key else out.error.strip()
+                progress(f"    {cause.splitlines()[0][:160]}")
+        ok = {n: o.pages for n, o in doc.outputs.items() if o.status == "ok"}
+        doc.metrics = metrics.compute_doc_metrics(doc.pages, layer, ok)
+
+    report = RunReport(
+        docs=reports, parsers=[s.name for s in specs], skipped=skipped,
+        thresholds=options.thresholds, demo=options.demo,
+    )
+    report.findings = findings.build_findings(report)
+    return report

parseoff/samples.py

@@ -0,0 +1,164 @@
+"""Deterministic synthetic PDFs used by `parseoff demo`.
+
+They are practice documents written for this tool (original English sentences, no real data).
+Each one reproduces one well-known parser pitfall, so the demo shows real parsers misbehaving
+on purpose-built input rather than replaying fake output:
+
+* report-table.pdf  - a grid table plus a running header/footer on every page.
+* two-column.pdf    - two text columns whose right column is drawn first in the content stream.
+* scanned-like.pdf  - a text page followed by a page that holds only shapes and a raster image.
+
+reportlab's `invariant=1` fixes creation dates and document ids, so the bytes are reproducible.
+"""
+
+from __future__ import annotations
+
+import textwrap
+from pathlib import Path
+
+from PIL import Image
+from reportlab.lib.pagesizes import A4
+from reportlab.lib.utils import ImageReader
+from reportlab.pdfgen.canvas import Canvas
+
+SAMPLE_NAMES = ("report-table.pdf", "two-column.pdf", "scanned-like.pdf")
+PAGE_COUNTS = {"report-table.pdf": 3, "two-column.pdf": 2, "scanned-like.pdf": 2}
+
+_W, _H = A4
+_LEFT = 56
+_BODY = "Helvetica"
+
+_PARAGRAPH = (
+    "This practice document was written for the parseoff demo. It contains plain English "
+    "sentences so that every parser has something to read, and a few structures that "
+    "parsers handle differently from one another."
+)
+_NOTES = (
+    "Section notes follow here. Parsers that read the content stream in drawing order, "
+    "parsers that sort text by position and parsers that build layout boxes can disagree "
+    "about the order of the same words."
+)
+
+
+def _canvas(path: Path) -> Canvas:
+    # invariant=1 -> no timestamps or random ids in the file, so the bytes are reproducible
+    return Canvas(str(path), pagesize=A4, invariant=1, pageCompression=0)
+
+
+def _lines(c: Canvas, x: float, y: float, text: str, width: int = 88, size: int = 10) -> float:
+    """Draw wrapped text from (x, y) downwards; return the next free y."""
+    c.setFont(_BODY, size)
+    for line in textwrap.wrap(text, width=width):
+        c.drawString(x, y, line)
+        y -= size + 4
+    return y
+
+
+def _frame(c: Canvas, page: int, total: int) -> None:
+    """Running header and page-number footer, identical on every page (boilerplate lesson)."""
+    c.setFont(_BODY, 9)
+    c.drawString(_LEFT, _H - 36, "ACME Handbook - draft")
+    c.drawString(_LEFT, 30, f"Page {page} of {total}")
+
+
+def _report_table(path: Path) -> None:
+    c = _canvas(path)
+    total = 3
+    # page 1: title, headings, paragraph, a 3 x 4 grid table
+    _frame(c, 1, total)
+    c.setFont("Helvetica-Bold", 20)
+    c.drawString(_LEFT, _H - 90, "Quarterly Report")
+    c.setFont("Helvetica-Bold", 14)
+    c.drawString(_LEFT, _H - 125, "1. Overview")
+    y = _lines(c, _LEFT, _H - 145, _PARAGRAPH)
+    c.setFont("Helvetica-Bold", 14)
+    c.drawString(_LEFT, y - 16, "2. Numbers")
+    rows = [
+        ["Region", "Q1", "Q2"],
+        ["North", "120", "135"],
+        ["South", "98", "101"],
+        ["East", "77", "90"],
+    ]
+    col_w, row_h, top = 110, 24, y - 40
+    c.setLineWidth(0.8)
+    for r, row in enumerate(rows):
+        for k, cell in enumerate(row):
+            x = _LEFT + k * col_w
+            c.rect(x, top - (r + 1) * row_h, col_w, row_h)  # grid lines -> a ruled table
+            c.setFont(_BODY, 10)
+            c.drawString(x + 8, top - (r + 1) * row_h + 8, cell)
+    c.showPage()
+    # pages 2 and 3: a heading and a paragraph each
+    for page, heading in ((2, "3. Method"), (3, "4. Notes")):
+        _frame(c, page, total)
+        c.setFont("Helvetica-Bold", 14)
+        c.drawString(_LEFT, _H - 90, heading)
+        _lines(c, _LEFT, _H - 112, _NOTES if page == 3 else _PARAGRAPH + " " + _NOTES)
+        c.showPage()
+    c.save()
+
+
+_WORDS = (
+    "river stone lantern copper meadow signal harbor velvet orchid timber anchor ember "
+    "falcon quartz willow marble thistle cobalt saffron juniper pebble zephyr mosaic"
+).split()
+
+
+def _column_line(tag: str, i: int) -> str:
+    # three distinct words per line so reordering is visible to a word-order comparison
+    w = [_WORDS[(i * 3 + k) % len(_WORDS)] for k in range(3)]
+    return f"{tag}{i:02d} {w[0]} {w[1]} {w[2]}"
+
+
+def _two_column(path: Path) -> None:
+    c = _canvas(path)
+    left_x, right_x, top, step = _LEFT, 320, _H - 100, 18
+    c.setFont("Helvetica-Bold", 14)
+    # page 1: two columns; the RIGHT column is drawn first, so stream order is R01..R12, L01..L12
+    for tag, x in (("R", right_x), ("L", left_x)):
+        c.setFont(_BODY, 10)
+        for i in range(1, 13):
+            c.drawString(x, top - (i - 1) * step, _column_line(tag, i))
+    c.showPage()
+    # page 2: an ordinary single column, where every parser should agree
+    c.setFont("Helvetica-Bold", 14)
+    c.drawString(_LEFT, _H - 90, "Single column page")
+    _lines(c, _LEFT, _H - 112, (_PARAGRAPH + " " + _NOTES) * 2)
+    c.showPage()
+    c.save()
+
+
+def _scanned_like(path: Path) -> None:
+    c = _canvas(path)
+    c.setFont("Helvetica-Bold", 14)
+    c.drawString(_LEFT, _H - 90, "Scanned appendix")
+    _lines(c, _LEFT, _H - 112, "This first page has a real text layer. " + _PARAGRAPH)
+    c.showPage()
+    # page 2: no text at all - a raster image of a "scan" and some shapes
+    img = Image.new("L", (160, 220), 255)
+    pix = img.load()
+    assert pix is not None
+    for yy in range(220):
+        for xx in range(160):
+            if (yy // 20) % 2 == 0 and 12 < xx < 148:  # horizontal "text bars"
+                pix[xx, yy] = 40 if (xx // 6 + yy // 4) % 3 else 140
+    c.drawImage(ImageReader(img), _LEFT, _H - 330, width=320, height=440)
+    c.rect(_LEFT - 8, _H - 340, 340, 460)
+    c.showPage()
+    c.save()
+
+
+def create_samples(out_dir: Path) -> list[Path]:
+    """Write the three sample PDFs into `out_dir` and return their paths (sorted by name)."""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    builders = {
+        "report-table.pdf": _report_table,
+        "two-column.pdf": _two_column,
+        "scanned-like.pdf": _scanned_like,
+    }
+    paths = []
+    for name in SAMPLE_NAMES:
+        path = out_dir / name
+        builders[name](path)
+        paths.append(path)
+    return sorted(paths)

parseoff/textlayer.py

@@ -0,0 +1,46 @@
+"""PDF text-layer baseline and page images via pypdfium2 (BSD-3 / Apache-2.0)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def page_count(pdf_path: Path) -> int:
+    """Number of pages; raises an exception with a readable message if the PDF cannot be opened."""
+    import pypdfium2 as pdfium
+
+    pdf = pdfium.PdfDocument(str(pdf_path))
+    try:
+        return len(pdf)
+    finally:
+        pdf.close()
+
+
+def layer_texts(pdf_path: Path, pages: list[int]) -> list[str]:
+    """Text layer per requested page. An empty string means the page has no extractable text."""
+    import pypdfium2 as pdfium
+
+    pdf = pdfium.PdfDocument(str(pdf_path))
+    try:
+        out = []
+        for number in pages:
+            textpage = pdf[number - 1].get_textpage()
+            out.append(textpage.get_text_range())
+            textpage.close()
+        return out
+    finally:
+        pdf.close()
+
+
+def render_pages(pdf_path: Path, pages: list[int], out_dir: Path, prefix: str, scale: float = 1.0) -> None:
+    """Write ``<prefix>-p<N>.png`` for each page (low resolution on purpose: reports stay small)."""
+    import pypdfium2 as pdfium
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    pdf = pdfium.PdfDocument(str(pdf_path))
+    try:
+        for number in pages:
+            image = pdf[number - 1].render(scale=scale).to_pil()
+            image.save(out_dir / f"{prefix}-p{number}.png")
+    finally:
+        pdf.close()

parseoff/worker.py

@@ -0,0 +1,63 @@
+"""Subprocess entry point: ``python -m parseoff.worker`` runs ONE parser on ONE PDF.
+
+Running each (document, parser) pair in its own process isolates crashes, native-library noise on
+stderr and hangs - the parent enforces a timeout and simply kills the process.
+The result is written as JSON to ``--out``; a parser exception is captured into the JSON, not raised.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+
+def _version(parser: str, custom: bool) -> str:
+    if custom:
+        return "custom"
+    from .registry import installed_version
+
+    return installed_version(parser) or ""
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="python -m parseoff.worker")
+    ap.add_argument("--pdf", required=True)
+    ap.add_argument("--pages", required=True, help="comma separated 1-based page numbers")
+    ap.add_argument("--out", required=True)
+    group = ap.add_mutually_exclusive_group(required=True)
+    group.add_argument("--parser")
+    group.add_argument("--custom", help="NAME=module:function")
+    args = ap.parse_args(argv)
+
+    pages = [int(p) for p in args.pages.split(",") if p]
+    result: dict = {"pages": [], "seconds": 0.0, "error": "", "version": ""}
+    started = time.perf_counter()
+    try:
+        if args.custom:
+            from .adapters.custom import load, parse_spec
+
+            spec = parse_spec(args.custom)
+            convert, name = load(spec), spec.name
+        else:
+            from .adapters import get_adapter
+
+            convert, name = get_adapter(args.parser), args.parser
+        result["version"] = _version(name, bool(args.custom))
+        texts = convert(args.pdf, pages)
+        if not isinstance(texts, list) or not all(isinstance(t, str) for t in texts):
+            raise TypeError("a parser must return a list of strings, one per requested page")
+        if len(texts) != len(pages):
+            raise ValueError(f"parser returned {len(texts)} page(s) but {len(pages)} were requested")
+        result["pages"] = texts
+    except Exception as exc:  # noqa: BLE001 - every parser failure must be reported, not raised
+        result["error"] = f"{type(exc).__name__}: {exc}"
+    result["seconds"] = round(time.perf_counter() - started, 3)
+    Path(args.out).write_text(json.dumps(result, ensure_ascii=True), encoding="utf-8")
+    return 1 if result["error"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())