inkgrid 0.1.0__py3-none-any.whl

inkgrid/__init__.py

@@ -0,0 +1,100 @@
+"""inkgrid: exact tables and text from born-digital PDFs.
+
+`read()` turns a PDF into a validated `Document`; `verify()` grades it against its PDF with a second
+engine; `read_pages()` returns the raw page model, for debugging a reading.
+"""
+
+import importlib.metadata
+
+from inkgrid.api import read, read_pages, verify
+from inkgrid.errors import (
+    InkgridError,
+    InvariantError,
+    PasswordRequired,
+    PdfOpenError,
+    SourceMismatch,
+    StrictModeError,
+    WrongPassword,
+)
+from inkgrid.model import (
+    Block,
+    Cell,
+    Defect,
+    DefectCode,
+    Definition,
+    Document,
+    Finding,
+    FindingCode,
+    Footnote,
+    Furniture,
+    Grid,
+    Heading,
+    Interval,
+    Ledger,
+    Lexicon,
+    Link,
+    LinkEnd,
+    ListItem,
+    PageCheck,
+    PageInfo,
+    PageModel,
+    Paragraph,
+    Producer,
+    Profile,
+    Reading,
+    Rect,
+    Region,
+    Rule,
+    Severity,
+    Table,
+    VerificationReport,
+    Word,
+)
+
+__version__ = importlib.metadata.version("inkgrid")
+
+__all__ = [
+    "Block",
+    "Cell",
+    "Defect",
+    "DefectCode",
+    "Definition",
+    "Document",
+    "Finding",
+    "FindingCode",
+    "Footnote",
+    "Furniture",
+    "Grid",
+    "Heading",
+    "InkgridError",
+    "Interval",
+    "InvariantError",
+    "Ledger",
+    "Lexicon",
+    "Link",
+    "LinkEnd",
+    "ListItem",
+    "PageCheck",
+    "PageInfo",
+    "PageModel",
+    "Paragraph",
+    "PasswordRequired",
+    "PdfOpenError",
+    "Producer",
+    "Profile",
+    "Reading",
+    "Rect",
+    "Region",
+    "Rule",
+    "Severity",
+    "SourceMismatch",
+    "StrictModeError",
+    "Table",
+    "VerificationReport",
+    "Word",
+    "WrongPassword",
+    "__version__",
+    "read",
+    "read_pages",
+    "verify",
+]

inkgrid/__main__.py

@@ -0,0 +1,5 @@
+"""`python -m inkgrid`: the same as the `inkgrid` command."""
+
+from inkgrid.cli import main
+
+raise SystemExit(main())

inkgrid/api.py

@@ -0,0 +1,167 @@
+"""The public entry points: the only module where the shell meets the core."""
+
+import importlib.metadata
+
+from inkgrid.core.pipeline import build_document
+from inkgrid.core.tables.pages import lattice_pages
+from inkgrid.errors import SourceMismatch, StrictModeError
+from inkgrid.model.canonical import sha256_hex
+from inkgrid.model.config import Lexicon, Profile
+from inkgrid.model.document import Document, Lattice
+from inkgrid.model.findings import Severity, summarize
+from inkgrid.model.page import Reading
+from inkgrid.model.verification import VerificationReport
+from inkgrid.read.camelot_reader import read_lattice
+from inkgrid.read.pymupdf_reader import lattice_copy, page_frames, read_pdf
+from inkgrid.read.source import SourceLike, load_source
+from inkgrid.verify.pdfium_reader import read_ink
+from inkgrid.verify.report import verify_document
+
+
+def read_pages(source: SourceLike, *, password: str | None = None) -> Reading:
+    """Read a PDF's text layer and drawings into the raw page model.
+
+    This is the debugging view of a reading: words, rules and page measurements, before any layout
+    decision. `read()` builds blocks on top of it.
+
+    Args:
+        source: a path, bytes-like object, or binary stream holding a PDF.
+        password: the user password, for an encrypted PDF.
+
+    Raises:
+        TypeError: `source` is not a supported input type.
+        PdfOpenError: the input is missing, empty, or not a readable PDF.
+        PasswordRequired: the PDF needs a password and none was given.
+        WrongPassword: the password does not open the PDF.
+    """
+    loaded = load_source(source)
+    return read_pdf(loaded.data, file_name=loaded.file_name, password=password)
+
+
+def _lattice(value: str) -> Lattice:
+    match value:
+        case "combined":
+            return "combined"
+        case "vector":
+            return "vector"
+        case "raster":
+            return "raster"
+        case _:
+            msg = f"lattice must be 'combined', 'vector', or 'raster', not {value!r}"
+            raise ValueError(msg)
+
+
+def _profile(value: object) -> Profile:
+    """The profile to run, revalidated: callers without type checking can pass anything."""
+    if value is None:
+        return Profile.default()
+    if not isinstance(value, Profile):
+        msg = f"profile must be an inkgrid.Profile, not {type(value).__name__}"
+        raise TypeError(msg)
+    return Profile.model_validate(value)
+
+
+def _lexicon(value: object) -> Lexicon:
+    """The lexicon to run, revalidated: callers without type checking can pass anything."""
+    if value is None:
+        return Lexicon.default()
+    if not isinstance(value, Lexicon):
+        msg = f"lexicon must be an inkgrid.Lexicon, not {type(value).__name__}"
+        raise TypeError(msg)
+    return Lexicon.model_validate(value)
+
+
+def read(
+    source: SourceLike,
+    *,
+    password: str | None = None,
+    lexicon: Lexicon | None = None,
+    profile: Profile | None = None,
+    lattice: str = "combined",
+    strict: bool = False,
+) -> Document:
+    """Read a PDF into a validated `Document`: every word in exactly one block, in reading order.
+
+    Args:
+        source: a path, bytes-like object, or binary stream holding a PDF.
+        password: the user password, for an encrypted PDF.
+        lexicon: token classes; `Lexicon.default()` when omitted.
+        profile: geometry tolerances; `Profile.default()` when omitted.
+        lattice: Camelot's lattice engine for ruled tables: `combined`, `vector`, or `raster`.
+        strict: raise instead of returning a document that carries an error-severity finding.
+
+    Raises:
+        TypeError: `source`, `lexicon`, or `profile` is of the wrong type.
+        ValueError: `lattice` is unknown, or a configuration fails revalidation.
+        PdfOpenError, PasswordRequired, WrongPassword: as for `read_pages`.
+        StrictModeError: `strict` is set and the document carries an error-severity finding.
+        InvariantError: a guarantee failed to hold, which is a bug in inkgrid.
+    """
+    checked_lexicon = _lexicon(lexicon)
+    checked_profile = _profile(profile)
+    checked_lattice = _lattice(lattice)
+    loaded = load_source(source)
+    reading = read_pdf(loaded.data, file_name=loaded.file_name, password=password)
+    # Camelot renders every page it reads at 300 DPI, so it reads only pages ruled both ways.
+    pages = lattice_pages(reading)
+    grids = (
+        read_lattice(
+            lattice_copy(loaded.data, password),
+            page_frames(loaded.data, password),
+            pages,
+            engine=checked_lattice,
+            password=None,
+        )
+        if pages
+        else None
+    )
+    doc = build_document(
+        reading,
+        lexicon=checked_lexicon,
+        profile=checked_profile,
+        lattice=checked_lattice,
+        grids=grids,
+    )
+    if strict and not doc.complete:
+        errors = [f for f in doc.findings if f.severity is Severity.ERROR]
+        msg = f"strict mode: the document carries error findings: {summarize(errors)}"
+        raise StrictModeError(msg)
+    return doc
+
+
+def _document(value: object) -> Document:
+    """The document to verify: callers without type checking can pass anything."""
+    if not isinstance(value, Document):
+        msg = f"doc must be an inkgrid.Document, not {type(value).__name__}"
+        raise TypeError(msg)
+    return value
+
+
+def verify(doc: Document, source: SourceLike, *, password: str | None = None) -> VerificationReport:
+    """Grade `doc` against the PDF it was read from, as a second engine (PDFium) reads it.
+
+    Every character PDFium reads must be owned by exactly one word that holds it, every table's
+    cells must hold the ink inside them, and every value must be bound to one block and one cell
+    (docs/specs/10-verify.md). A defect is reported, never raised.
+
+    Args:
+        doc: the document `read` returned for `source`.
+        source: a path, bytes-like object, or binary stream holding the same PDF.
+        password: the user password, for an encrypted PDF.
+
+    Raises:
+        TypeError: `doc` is not a `Document`, or `source` is not a supported input type.
+        SourceMismatch: `source` is not the PDF `doc` was read from.
+        PdfOpenError, PasswordRequired, WrongPassword: as for `read_pages`.
+    """
+    checked = _document(doc)
+    loaded = load_source(source)
+    digest = sha256_hex(loaded.data)
+    if digest != checked.source.sha256:
+        msg = (
+            f"the PDF's SHA-256 {digest[:16]}... differs from the document's "
+            f"{checked.source.sha256[:16]}...; it was read from another PDF"
+        )
+        raise SourceMismatch(msg)
+    ink = read_ink(loaded.data, password)
+    return verify_document(checked, ink, inkgrid_version=importlib.metadata.version("inkgrid"))

inkgrid/cli.py

@@ -0,0 +1,225 @@
+"""The `inkgrid` command line: `read`, `verify`, and `words`.
+
+`read` prints the `Document`, `verify` its report against the PDF, and `words` the page model. Exit
+codes: 0 success; 1 strict mode found an error-severity finding, or verification found a
+defect; 2 usage errors and unreadable input. Output is always UTF-8 bytes, whatever the console's
+encoding.
+"""
+
+import argparse
+import importlib.metadata
+import os
+import sys
+from collections import Counter
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+from inkgrid.api import read, read_pages, verify
+from inkgrid.errors import PasswordRequired, PdfOpenError, SourceMismatch, WrongPassword
+from inkgrid.model.document import Document, document_from_json
+from inkgrid.model.findings import Severity, summarize
+from inkgrid.model.page import Reading
+from inkgrid.model.verification import DefectCode, VerificationReport
+from inkgrid.render.inspector import build_inspector
+
+EXIT_OK = 0
+EXIT_STRICT = 1
+EXIT_DEFECTS = 1
+EXIT_USAGE = 2
+INPUT_ERRORS = (PdfOpenError, PasswordRequired, WrongPassword)
+
+
+class _UsageError(Exception):
+    """A problem with the arguments or the input: one line on stderr, exit 2."""
+
+
+def _add_common(command: argparse.ArgumentParser) -> None:
+    command.add_argument("pdf", type=Path, help="the PDF to read")
+    command.add_argument("-o", "--output", type=Path, help="write the JSON here instead of stdout")
+    command.add_argument("--pretty", action="store_true", help="indent the JSON for reading")
+    command.add_argument(
+        "--password-stdin",
+        action="store_true",
+        help="read the password from the first line of stdin (never from argv)",
+    )
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="inkgrid",
+        description="Exact tables and text from born-digital PDFs.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"inkgrid {importlib.metadata.version('inkgrid')}"
+    )
+    commands = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
+    read_command = commands.add_parser("read", help="print the Document (blocks, links) as JSON")
+    _add_common(read_command)
+    read_command.add_argument("--markdown", type=Path, help="also write the text as Markdown")
+    read_command.add_argument("--inspector", type=Path, help="also write the HTML inspector")
+    read_command.add_argument(
+        "--strict", action="store_true", help="exit 1 when an error-severity finding is present"
+    )
+    read_command.add_argument(
+        "--lattice",
+        choices=("combined", "vector", "raster"),
+        default="combined",
+        help="Camelot's engine for ruled tables (default: combined)",
+    )
+    verify_command = commands.add_parser(
+        "verify", help="grade a Document against its PDF; exit 1 on any defect"
+    )
+    _add_common(verify_command)
+    verify_command.add_argument(
+        "document", type=Path, help="the Document JSON `inkgrid read` wrote"
+    )
+    verify_command.add_argument(
+        "--inspector", type=Path, help="also write the HTML inspector with the defects drawn"
+    )
+    words = commands.add_parser("words", help="print the raw page model (words, rules) as JSON")
+    _add_common(words)
+    return parser
+
+
+def _password_from_stdin() -> str:
+    """The first stdin line, read as UTF-8 bytes, with only its line ending removed.
+
+    Raises:
+        UnicodeDecodeError: the line is not UTF-8.
+    """
+    buffer = getattr(sys.stdin, "buffer", None)
+    line: str = sys.stdin.readline() if buffer is None else buffer.readline().decode("utf-8")
+    return line.removesuffix("\n").removesuffix("\r")
+
+
+def _password(args: argparse.Namespace) -> str | None:
+    if not args.password_stdin:
+        return None
+    try:
+        return _password_from_stdin()
+    except UnicodeDecodeError as exc:
+        msg = "the password on stdin is not UTF-8"
+        raise _UsageError(msg) from exc
+
+
+def _write_stdout(data: bytes) -> None:
+    buffer = getattr(sys.stdout, "buffer", None)
+    if buffer is None:
+        sys.stdout.write(data.decode("utf-8"))
+        sys.stdout.flush()
+    else:
+        sys.stdout.flush()
+        buffer.write(data)
+        buffer.flush()
+
+
+def _discard_stdout() -> None:
+    """Point stdout at the null device, so the interpreter's final flush cannot fail again."""
+    devnull = os.open(os.devnull, os.O_WRONLY)
+    try:
+        os.dup2(devnull, sys.stdout.fileno())
+    finally:
+        os.close(devnull)
+
+
+def _write_file(path: Path, data: bytes) -> None:
+    try:
+        path.write_bytes(data)
+    except OSError as exc:
+        msg = f"cannot write {path}: {exc.strerror or exc}"
+        raise _UsageError(msg) from exc
+
+
+def _emit(model: Reading | Document | VerificationReport, args: argparse.Namespace) -> None:
+    text = model.model_dump_json(indent=2) if args.pretty else model.model_dump_json()
+    data = text.encode("utf-8") + b"\n"
+    if args.output is not None:
+        _write_file(args.output, data)
+        return
+    try:
+        _write_stdout(data)
+    except BrokenPipeError:
+        # The reader stopped early (`inkgrid read f.pdf | head`); what it read was correct.
+        _discard_stdout()
+
+
+def _words(args: argparse.Namespace) -> int:
+    _emit(read_pages(args.pdf, password=_password(args)), args)
+    return EXIT_OK
+
+
+def _read(args: argparse.Namespace) -> int:
+    password = _password(args)
+    doc = read(args.pdf, password=password, lattice=args.lattice)
+    _emit(doc, args)
+    if args.markdown is not None:
+        _write_file(args.markdown, doc.to_markdown().encode("utf-8"))
+    if args.inspector is not None:
+        try:
+            html = build_inspector(doc, args.pdf, password)
+        except ValueError as exc:
+            raise _UsageError(str(exc)) from exc
+        _write_file(args.inspector, html.encode("utf-8"))
+    if args.strict and not doc.complete:
+        errors = [f for f in doc.findings if f.severity is Severity.ERROR]
+        sys.stderr.write(f"inkgrid: strict mode: error findings: {summarize(errors)}\n")
+        return EXIT_STRICT
+    return EXIT_OK
+
+
+def _load_document(path: Path) -> Document:
+    try:
+        text = path.read_bytes()
+    except OSError as exc:
+        msg = f"cannot read {path}: {exc.strerror or exc}"
+        raise _UsageError(msg) from exc
+    try:
+        return document_from_json(text)
+    except ValueError as exc:
+        msg = f"{path} is not an inkgrid document: {exc}"
+        raise _UsageError(msg) from exc
+
+
+def _defect_summary(report: VerificationReport) -> str:
+    counts = Counter(d.code for d in report.defects)
+    total = len(report.defects)
+    codes = ", ".join(f"{code.value} {counts[code]}" for code in DefectCode if counts[code])
+    return f"{total} defect{'' if total == 1 else 's'}: {codes}"
+
+
+def _verify(args: argparse.Namespace) -> int:
+    password = _password(args)
+    doc = _load_document(args.document)
+    try:
+        report = verify(doc, args.pdf, password=password)
+    except SourceMismatch as exc:
+        raise _UsageError(str(exc)) from exc
+    _emit(report, args)
+    if args.inspector is not None:
+        html = build_inspector(doc, args.pdf, password, report)
+        _write_file(args.inspector, html.encode("utf-8"))
+    if not report.ok:
+        sys.stderr.write(f"inkgrid: verify: {_defect_summary(report)}\n")
+        return EXIT_DEFECTS
+    return EXIT_OK
+
+
+COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = {
+    "read": _read,
+    "verify": _verify,
+    "words": _words,
+}
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the command line and return its exit code."""
+    try:
+        args = _parser().parse_args(argv)
+    except SystemExit as exc:
+        code = exc.code
+        return code if isinstance(code, int) else (EXIT_OK if code is None else EXIT_USAGE)
+    try:
+        return COMMANDS[args.command](args)
+    except (*INPUT_ERRORS, _UsageError) as exc:
+        sys.stderr.write(f"inkgrid: {exc}\n")
+        return EXIT_USAGE

inkgrid/core/__init__.py

@@ -0,0 +1 @@
+"""The pipeline: pure functions over typed values. Imports only `inkgrid.model` and the stdlib."""