pdfdown 0.1.0__py3-none-any.whl

pdfdown/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

pdfdown/cleanup/__init__.py

@@ -0,0 +1,36 @@
+"""The cleanup layer: small steps that turn Docling's blocks into a clean article.
+
+Each step takes the Document and rewrites doc.blocks in place, adding a short
+plain-language note to doc.notes about what it changed. Order matters:
+text is rebuilt before captions are parsed, and paragraphs are stitched only
+after floats and furniture have been sorted out.
+"""
+
+from __future__ import annotations
+
+from pdfdown.cleanup import captions, figures, furniture, notes, order, retext, stitch
+from pdfdown.document import Document
+from pdfdown.pdftext import PdfText
+
+STEPS = [
+    ("furniture", furniture.run),
+    ("order", order.run),
+    ("figure text", figures.drop_inner_text),
+    ("retext", retext.run),
+    ("captions", captions.run),
+    ("icons", figures.drop_icons),
+    ("stitch", stitch.run),
+    ("notes", notes.run),
+]
+
+
+def clean(doc: Document, pdf: PdfText | None = None) -> Document:
+    own = pdf is None
+    pdf = pdf or PdfText(doc.source)
+    try:
+        for _name, step in STEPS:
+            step(doc, pdf)
+    finally:
+        if own:
+            pdf.close()
+    return doc

pdfdown/cleanup/captions.py

@@ -0,0 +1,126 @@
+"""Captions: join split pieces and attach each caption to its figure or table.
+
+In two-column journals a figure caption often runs across both columns and
+Docling returns it in pieces ("Fig. 1 | ... a, ...", then "c, ..."), with
+body text from the page in between. Pieces after a caption that starts with
+"Fig./Figure/Table N" on the same page, and that do not start a new caption,
+are merged into it. Each caption then goes to the nearest figure or table on
+its page, and its number names the image file (figure-2.png, table-1.png).
+"""
+
+from __future__ import annotations
+
+import re
+
+from pdfdown.document import Block, Document
+
+CAPTION_START = re.compile(r"^\s*(?:<[^>]+>)?\s*(Extended Data\s+)?(Fig(?:ure)?|Table|Supplementary Fig(?:ure)?)\.?\s*([A-Z]?\d+[a-z]?)\b", re.I)
+
+
+def number_of(text: str) -> tuple[str, str] | None:
+    """('figure', '2') for 'Fig. 2 | ...', ('table', '1') for 'Table 1.', ('figure', 'ED3') for 'Extended Data Fig. 3'."""
+    m = CAPTION_START.match(text)
+    if not m:
+        return None
+    kind = "table" if m.group(2).lower().startswith("table") else "figure"
+    prefix = "ED" if m.group(1) else ("S" if m.group(2).lower().startswith("supplementary") else "")
+    return kind, prefix + m.group(3)
+
+
+PROMOTE = re.compile(r"^\s*(?:Extended Data\s+)?(?:Supplementary\s+)?(?:Fig|Figure|Table)\.?\s*[A-Z]?\d+[a-z]?\s*[|.:]", re.I)
+
+
+def run(doc: Document, pdf=None) -> None:
+    blocks = doc.blocks
+    # 1. Promote "Fig. 2 |" and "Extended Data Fig. 3 |" text that Docling labelled as a paragraph
+    #    or a heading. The separator after the number keeps "Fig. 2 shows ..." in the body.
+    for b in blocks:
+        if b.kind in ("paragraph", "heading") and PROMOTE.match(b.text):
+            b.kind = "caption"
+    # 2. Merge continuation pieces into the caption they belong to: unnumbered caption pieces on the
+    #    page, and a paragraph right after an unfinished caption that carries on in lower case.
+    merged = 0
+    i = 0
+    while i < len(blocks):
+        b = blocks[i]
+        if b.kind == "caption" and number_of(b.text):
+            j = i + 1
+            while j < len(blocks):
+                nxt = blocks[j]
+                carries_on = (j == i + 1 and nxt.kind == "paragraph" and nxt.text[:1].islower()
+                              and not re.search(r"[.!?:]\s*$", b.text))
+                if (nxt.kind == "caption" or carries_on) and nxt.page == b.page and not number_of(nxt.text):
+                    b.text = b.text.rstrip() + " " + nxt.text.lstrip()
+                    b.parts += nxt.parts
+                    del blocks[j]
+                    merged += 1
+                    continue
+                if nxt.kind in ("caption",) or nxt.page != b.page:
+                    break
+                j += 1
+        i += 1
+    # 3. Attach captions: first by Docling's own link between a float and its caption,
+    #    then, for captions without one, to the nearest float of the matching kind on the page.
+    floats = [b for b in blocks if b.kind in ("figure", "table")]
+    attached = 0
+    by_ref = {b.extra.get("ref"): b for b in blocks if b.kind == "caption"}
+    for f in floats:
+        caps = [by_ref[r] for r in f.extra.get("caption_refs", []) if r in by_ref and not by_ref[r].extra.get("attached")]
+        if caps:
+            f.extra["caption"] = " ".join(c.text for c in caps)
+            info = number_of(caps[0].text)
+            if info:
+                f.extra["number"] = info[1]
+            for c in caps:
+                c.extra["attached"] = True
+                attached += 1
+    for cap in [b for b in blocks if b.kind == "caption"]:
+        info = number_of(cap.text)
+        want = info[0] if info else None
+        candidates = [f for f in floats if f.page == cap.page and "caption" not in f.extra and (want is None or f.kind == want)]
+        if not candidates:
+            continue
+        target = min(candidates, key=lambda f: _distance(f, cap))
+        target.extra["caption"] = cap.text
+        if info:
+            target.extra["number"] = info[1]
+        cap.extra["attached"] = True
+        attached += 1
+    attached_text = {_norm(f.extra["caption"])[:60] for f in floats if f.extra.get("caption")}
+    before = len(blocks)
+    doc.blocks = [b for b in blocks if not (b.kind == "caption" and b.extra.get("attached"))
+                  and not (b.kind in ("paragraph", "caption") and _norm(b.text)[:60] in attached_text)]
+    used = sum(1 for b in blocks if b.kind == "caption" and b.extra.get("attached"))  # one caption can serve several panels
+    duplicates = before - len(doc.blocks) - used
+    _number_rest(doc)
+    captioned = sum(1 for f in floats if f.extra.get("caption"))
+    if merged:
+        doc.notes.append(f"Joined {merged} captions that were split into pieces.")
+    if captioned:
+        doc.notes.append(f"Attached captions to {captioned} figures and tables.")
+    if duplicates:
+        doc.notes.append(f"Removed {duplicates} duplicate copies of captions.")
+
+
+def _distance(f: Block, cap: Block) -> float:
+    """Vertical gap between a float and a caption, preferring overlap in columns."""
+    gap = max(cap.box.t - f.box.b, f.box.t - cap.box.b, 0)
+    shared = f.box.overlap_x(cap.box) / max(min(f.box.width, cap.box.width), 1)
+    return gap + (0 if shared > 0.3 else 500)
+
+
+def _number_rest(doc: Document) -> None:
+    """Give every figure and table without a caption number the next free one."""
+    for kind in ("figure", "table"):
+        used = {b.extra.get("number") for b in doc.blocks if b.kind == kind}
+        n = 0
+        for b in doc.blocks:
+            if b.kind == kind and not b.extra.get("number"):
+                n += 1
+                while f"u{n}" in used:
+                    n += 1
+                b.extra["number"] = f"u{n}"  # 'u' marks an unnumbered figure in file names
+
+
+def _norm(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "", re.sub(r"<[^>]+>", "", text.lower()))

pdfdown/cleanup/figures.py

@@ -0,0 +1,62 @@
+"""Figures: drop text that belongs inside a figure (axis labels, legends, panel letters).
+
+Docling sometimes reports the labels drawn inside a vector chart as separate
+text blocks ("Time Time" under Nature's Fig. 1). Any short text block that
+lies mostly inside a figure on the same page is part of the picture, which
+pdfdown crops as an image anyway.
+"""
+
+from __future__ import annotations
+
+from pdfdown.document import Box, Document
+
+
+def _inside_share(inner: Box, outer: Box) -> float:
+    w = max(0.0, min(inner.r, outer.r) - max(inner.l, outer.l))
+    h = max(0.0, min(inner.b, outer.b) - max(inner.t, outer.t))
+    area = max(inner.width * inner.height, 1e-6)
+    return w * h / area
+
+
+def drop_inner_text(doc: Document, pdf=None) -> None:
+    figures = [b for b in doc.blocks if b.kind == "figure" and b.box]
+    kept, dropped = [], 0
+    for b in doc.blocks:
+        if b.kind in ("paragraph", "list_item", "heading") and b.box and len(b.text) < 200:
+            if any(f.page == b.page and _inside_share(b.box, f.box) > 0.6 for f in figures):
+                dropped += 1
+                continue
+            if _label_beside_figure(b, figures):
+                dropped += 1
+                continue
+        kept.append(b)
+    doc.blocks = kept
+    if dropped:
+        doc.notes.append(f"Dropped {dropped} text fragments drawn inside figures (axis labels and legends).")
+
+
+def _label_beside_figure(b, figures) -> bool:
+    """A few words with no sentence punctuation, right at a figure's edge: an axis title or legend."""
+    words = b.text.split()
+    if b.kind != "paragraph" or len(words) > 4 or len(b.text) > 40 or b.text.rstrip().endswith((".", ":", ";")):
+        return False
+    for f in figures:
+        if f.page != b.page or b.box.overlap_x(f.box) < 0.5 * b.box.width:
+            continue
+        gap = max(b.box.t - f.box.b, f.box.t - b.box.b, 0)
+        if gap < 25:
+            return True
+    return False
+
+
+ICON = 24.0  # points; a figure smaller than this both ways is a logo or badge
+
+
+def drop_icons(doc: Document, pdf=None) -> None:
+    """Drop tiny images without a caption, such as a journal's "Check for updates" badge."""
+    keep = [b for b in doc.blocks if not (b.kind == "figure" and not b.extra.get("caption")
+                                          and b.box.width < ICON and b.box.height < ICON)]
+    dropped = len(doc.blocks) - len(keep)
+    doc.blocks = keep
+    if dropped:
+        doc.notes.append(f"Dropped {dropped} tiny image{'s' * (dropped != 1)} without a caption (logos and badges).")

pdfdown/cleanup/furniture.py

@@ -0,0 +1,49 @@
+"""Remove page furniture: running headers and footers, page numbers, journal lines.
+
+Docling marks most of them, but not all: Nature's running header "Article"
+comes through as a section heading on every left-hand page. A block is
+treated as furniture when its text (with digits ignored) repeats on at least
+a third of the pages, near the top or bottom edge.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+
+from pdfdown.document import Document
+
+EDGE = 0.09  # top or bottom 9% of the page
+
+
+def _key(text: str) -> str:
+    return re.sub(r"\d+", "#", text.strip().lower())
+
+
+def run(doc: Document, pdf=None) -> None:
+    pages = max(len(doc.page_sizes), 1)
+    seen: dict[str, set[int]] = defaultdict(set)
+    for b in doc.blocks:
+        if b.box and b.text and len(b.text) < 120 and _near_edge(doc, b):
+            seen[_key(b.text)].add(b.page)
+    repeated = {k for k, ps in seen.items() if len(ps) >= max(2, pages / 3)}
+    words = {k for k in repeated if k.replace("#", "").strip() and len(k.split()) <= 3}
+    kept, dropped = [], 0
+    for b in doc.blocks:
+        if b.kind == "heading":
+            for key in words:  # "Article Methods": a running header merged into a heading
+                if _key(b.text).startswith(key + " ") and len(b.text) > len(key) + 2:
+                    b.text = b.text[len(key):].strip()
+        page_number = bool(re.fullmatch(r"\d{1,4}", b.text)) and b.box is not None and _near_edge(doc, b)
+        if b.kind == "furniture" or page_number or (b.text and _key(b.text) in repeated and _near_edge(doc, b)):
+            dropped += 1
+            continue
+        kept.append(b)
+    doc.blocks = kept
+    if dropped:
+        doc.notes.append(f"Removed {dropped} page headers, footers and page numbers.")
+
+
+def _near_edge(doc: Document, b) -> bool:
+    height = doc.page_sizes.get(b.page, (0, 792))[1]
+    return b.box.t < EDGE * height or b.box.b > (1 - EDGE) * height

pdfdown/cleanup/notes.py

@@ -0,0 +1,25 @@
+"""Footnotes and affiliations: move them out of the reading flow to a Notes section.
+
+On the first page of an article the author affiliations sit at the bottom of
+the first column; left in place they interrupt the introduction. They are
+moved to the end, before the references when there is a references heading.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pdfdown.document import Block, Document
+
+REFERENCES = re.compile(r"^\s*(references|bibliography|literature cited)\s*$", re.I)
+
+
+def run(doc: Document, pdf=None) -> None:
+    notes = [b for b in doc.blocks if b.kind == "footnote"]
+    if not notes:
+        return
+    body = [b for b in doc.blocks if b.kind != "footnote"]
+    section = [Block(kind="heading", text="Notes", level=2, extra={"generated": True})] + notes
+    refs = next((i for i, b in enumerate(body) if b.kind == "heading" and REFERENCES.match(b.text)), None)
+    doc.blocks = body[:refs] + section + body[refs:] if refs is not None else body + section
+    doc.notes.append(f"Moved {len(notes)} footnotes and affiliations to a Notes section at the end.")

pdfdown/cleanup/order.py

@@ -0,0 +1,119 @@
+"""Reading order on two-column pages, and author lines mistaken for headings.
+
+The layout step usually reads two columns correctly, but not always: on the
+first page of the LLaMA paper it read the right column before the abstract in
+the left one. This step puts every two-column page into the order a person
+reads it. Blocks that cross the middle of the page (a title, an author list, a
+wide figure) split the page into bands; within each band the left column is
+read from top to bottom, then the right column. Pages with no block wholly in
+each column are left as they are.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pdfdown.document import Block, Document
+
+TEXT = {"title", "heading", "paragraph", "list_item", "caption", "footnote", "reference", "code", "equation"}
+STRADDLE = 4.0  # points a block must reach past the middle of the page, on both sides, to count as full width
+
+
+def run(doc: Document, pdf=None) -> None:
+    pages: dict[int, list[int]] = {}
+    for i, b in enumerate(doc.blocks):
+        if b.kind != "furniture" and b.box is not None:
+            pages.setdefault(b.page, []).append(i)
+    changed = 0
+    for page, positions in pages.items():
+        width = doc.page_sizes.get(page, (612.0, 792.0))[0]
+        blocks = [doc.blocks[i] for i in positions]
+        ordered = _read_order(blocks, width / 2)
+        if ordered is not blocks and ordered != blocks:
+            for i, b in zip(positions, ordered):
+                doc.blocks[i] = b
+            changed += 1
+    _join_split_title(doc)
+    _authors_are_not_headings(doc)
+    if changed:
+        doc.notes.append(f"Put {changed} two-column page{'s' * (changed != 1)} into reading order (left column, then right).")
+
+
+def _side(b: Block, mid: float) -> str:
+    if b.box.l < mid - STRADDLE and b.box.r > mid + STRADDLE:
+        return "full"
+    return "left" if (b.box.l + b.box.r) / 2 < mid else "right"
+
+
+def _two_columns(blocks: list[Block], sides: dict[int, str]) -> bool:
+    """At least two text blocks in each column, and columns of similar width (a narrow sidebar is not a column)."""
+    left = [b.box.width for b in blocks if b.kind in TEXT and sides[id(b)] == "left"]
+    right = [b.box.width for b in blocks if b.kind in TEXT and sides[id(b)] == "right"]
+    if len(left) < 2 or len(right) < 2:
+        return False
+    ratio = max(left) / max(max(right), 1.0)
+    return 0.7 <= ratio <= 1.4
+
+
+def _right_before_left(blocks: list[Block], sides: dict[int, str]) -> bool:
+    """Did the layout step read right-column text before left-column text that sits beside it?"""
+    for i, a in enumerate(blocks):
+        if sides[id(a)] != "right" or a.kind not in TEXT:
+            continue
+        for b in blocks[i + 1:]:
+            if sides[id(b)] == "left" and b.kind in TEXT and b.box.t < a.box.b:  # b is not wholly below a
+                return True
+    return False
+
+
+def _read_order(blocks: list[Block], mid: float) -> list[Block]:
+    sides = {id(b): _side(b, mid) for b in blocks}
+    if not _two_columns(blocks, sides) or not _right_before_left(blocks, sides):
+        return blocks  # one column, a sidebar layout, or already in reading order: keep the layout step's order
+    full = sorted((b for b in blocks if sides[id(b)] == "full"), key=lambda b: b.box.t)
+    columns = [b for b in blocks if sides[id(b)] != "full"]
+    out: list[Block] = []
+    top = float("-inf")
+    for separator in [*full, None]:
+        bottom = separator.box.t if separator else float("inf")
+        band = [b for b in columns if top <= (b.box.t + b.box.b) / 2 < bottom]
+        out += sorted((b for b in band if sides[id(b)] == "left"), key=lambda b: b.box.t)
+        out += sorted((b for b in band if sides[id(b)] == "right"), key=lambda b: b.box.t)
+        if separator:
+            out.append(separator)
+            top = (separator.box.t + separator.box.b) / 2
+    missing = [b for b in blocks if b not in out]  # never lose a block, whatever its position
+    return out + missing
+
+
+def _join_split_title(doc: Document) -> None:
+    """'Direct Preference Optimization:' and the line under it are one title, not a title and a heading."""
+    first = next((i for i, b in enumerate(doc.blocks) if b.page == 1 and b.kind in ("title", "heading")), None)
+    if first is None or first + 1 >= len(doc.blocks):
+        return
+    title, nxt = doc.blocks[first], doc.blocks[first + 1]
+    unfinished = re.search(r"[:\-–—,]\s*$|\b(of|and|for|the|in|on|with|to|a|an)\s*$", title.text, re.I)
+    close = nxt.box is not None and title.box is not None and 0 <= nxt.box.t - title.box.b < title.box.height
+    if nxt.page == 1 and nxt.kind == "heading" and unfinished and close:
+        title.text = f"{title.text.rstrip()} {nxt.text.strip()}"
+        title.box = title.box.union(nxt.box)
+        title.parts += nxt.parts
+        del doc.blocks[first + 1]
+
+
+_NAME_LIST = re.compile(r"(?:[A-Z][\w’'.-]+\s+){1,3}[A-Z][\w’'.-]+(?:<sup>[^<]*</sup>|[∗*†‡§¶\d,])*\s*,")
+
+
+def _authors_are_not_headings(doc: Document) -> None:
+    """On the first page, a 'heading' that is a list of names before the first real section is the author line."""
+    seen_title = False
+    for b in doc.blocks:
+        if b.page != 1:
+            break
+        if b.kind in ("title", "heading") and not seen_title:
+            seen_title = True
+            continue
+        if b.kind == "heading" and re.match(r"^\s*(abstract|\d|[IVX]+\.|introduction)", b.text, re.I):
+            break
+        if b.kind == "heading" and len(_NAME_LIST.findall(b.text)) >= 2:
+            b.kind = "paragraph"

pdfdown/cleanup/retext.py

@@ -0,0 +1,34 @@
+"""Rebuild block text from the PDF's characters (superscripts, subscripts, hyphenation).
+
+Docling's text is kept when the PDF has no usable character layer for the
+block (a scanned page) or when the rebuilt text is clearly incomplete.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pdfdown.document import Document
+from pdfdown.pdftext import PdfText, block_text, vocabulary
+
+REBUILT = {"title", "heading", "paragraph", "list_item", "caption", "footnote", "reference"}
+
+
+def _plain(text: str) -> str:
+    return re.sub(r"</?su[bp]>|\s+", "", text)
+
+
+def run(doc: Document, pdf: PdfText) -> None:
+    vocab = vocabulary(pdf, sorted(doc.page_sizes))
+    rebuilt = 0
+    for b in doc.blocks:
+        if b.kind not in REBUILT or not b.parts:
+            continue
+        text, seen = block_text(pdf, b.parts, vocab)
+        old = re.sub(r"\s+", "", b.text)
+        # Accept the rebuilt text when it covers roughly the same characters as Docling's.
+        if seen and old and 0.85 <= len(_plain(text)) / max(len(old), 1) <= 1.2:
+            b.text = text
+            rebuilt += 1
+    if rebuilt:
+        doc.notes.append(f"Rebuilt {rebuilt} text blocks from the PDF's characters, restoring superscripts and subscripts and joining lines.")

pdfdown/cleanup/stitch.py

@@ -0,0 +1,63 @@
+"""Stitch paragraphs that a column break, page break or figure interrupted.
+
+PDFs end a block wherever the column or page ends, so a sentence like
+"... on the basis of" / "41,298,433 papers ..." arrives as two paragraphs,
+sometimes with a figure, caption or footnote between them. A paragraph that
+does not end a sentence is joined with the next body paragraph; anything that
+sat between them (figures, tables, footnotes) moves to just after the joined
+paragraph, so floats still appear next to the text that surrounds them.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pdfdown.document import Document
+
+END = re.compile(r"""[.!?:;](?:["'”’)\]]|</sup>|\s*<sup>[^<]*</sup>)*\s*$""")
+FLOATING = {"figure", "table", "footnote", "caption"}
+
+
+def ends_sentence(text: str) -> bool:
+    return bool(END.search(text))
+
+
+def run(doc: Document, pdf=None) -> None:
+    out = []
+    joined = 0
+    i = 0
+    blocks = doc.blocks
+    while i < len(blocks):
+        b = blocks[i]
+        if b.kind != "paragraph":
+            out.append(b)
+            i += 1
+            continue
+        held = []
+        while not ends_sentence(b.text):
+            j = i + 1
+            between = []
+            while j < len(blocks) and blocks[j].kind in FLOATING:
+                between.append(blocks[j])
+                j += 1
+            if j < len(blocks) and blocks[j].kind == "paragraph" and _continues(blocks[j].text):
+                nxt = blocks[j]
+                b.text = b.text.rstrip() + " " + nxt.text.lstrip()
+                b.parts = b.parts + nxt.parts
+                held += between
+                i = j
+                joined += 1
+            else:
+                break
+        out.append(b)
+        out.extend(held)
+        i += 1
+    doc.blocks = out
+    if joined:
+        doc.notes.append(f"Joined {joined} paragraphs split across columns, pages or figures.")
+
+
+def _continues(text: str) -> bool:
+    """A continuation starts in lowercase, with a digit, or with punctuation, never with a new heading-like line."""
+    stripped = re.sub(r"^<su[bp]>[^<]*</su[bp]>", "", text.lstrip())
+    return bool(stripped) and (stripped[0].islower() or stripped[0].isdigit() or stripped[0] in ",;:)]–-")

pdfdown/cli.py

@@ -0,0 +1,128 @@
+"""Command line: `pdfdown` opens the TUI; `pdfdown --batch a.pdf b.pdf` converts without it.
+
+    pdfdown                          open the TUI and pick PDFs there
+    pdfdown paper.pdf more/*.pdf     open the TUI with these PDFs queued
+    pdfdown --batch paper.pdf        convert in the terminal, no TUI (also when output is piped)
+    pdfdown check out/paper/paper.md list problems left in a converted file
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from pathlib import Path
+
+from pdfdown import __version__
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="pdfdown",
+        description="Convert PDFs (research articles first) to clean Markdown, with figures, tables and equations.",
+        epilog="Example: pdfdown --batch paper.pdf -o notes/  writes notes/paper/paper.md and its images.",
+    )
+    p.add_argument("pdfs", nargs="*", type=Path, help="PDF files or folders of PDFs")
+    p.add_argument("-o", "--out", type=Path, default=Path("pdfdown-output"),
+                   help="output folder; each PDF gets its own subfolder (default: ./pdfdown-output)")
+    p.add_argument("--batch", action="store_true", help="convert without the TUI")
+    p.add_argument("--no-latex", action="store_true", help="keep every equation as an image (skips the 97 MB equation model)")
+    p.add_argument("--ocr", action="store_true", help="run OCR, for scanned PDFs without a text layer")
+    p.add_argument("--no-tables", action="store_true", help="crop tables as images instead of rebuilding them")
+    p.add_argument("--device", default="auto", choices=["auto", "cpu", "mps", "cuda"], help="where the layout models run")
+    p.add_argument("--version", action="version", version=f"pdfdown {__version__}")
+    return p
+
+
+def expand(paths: list[Path]) -> list[Path]:
+    out: list[Path] = []
+    for path in paths:
+        path = path.expanduser()
+        if path.is_dir():
+            out += sorted(p for p in path.rglob("*") if p.suffix.lower() == ".pdf")
+        else:
+            out.append(path)
+    return out
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    if argv[:1] == ["check"]:
+        return _check(argv[1:])
+    args = _parser().parse_args(argv)
+    from pdfdown.engine import EngineOptions
+
+    options = EngineOptions(tables=not args.no_tables, ocr=args.ocr, device=args.device)
+    pdfs = expand(args.pdfs)
+    missing = [p for p in pdfs if not p.is_file()]
+    if missing:
+        print(f"pdfdown: no such file: {missing[0]}", file=sys.stderr)
+        return 2
+    if args.batch or not sys.stdout.isatty():
+        if not pdfs:
+            print("pdfdown: give at least one PDF with --batch", file=sys.stderr)
+            return 2
+        return _batch(pdfs, args.out, not args.no_latex, options)
+    from pdfdown.tui.app import PdfdownApp
+
+    PdfdownApp(pdfs, args.out, latex=not args.no_latex, options=options).run()
+    return 0
+
+
+def _batch(pdfs: list[Path], out: Path, latex: bool, options) -> int:
+    from pdfdown.pipeline import run
+
+    model = None
+    failed = 0
+    for n, pdf in enumerate(pdfs, start=1):
+        start = time.time()
+        print(f"[{n}/{len(pdfs)}] {pdf.name}", flush=True)
+        try:
+            if latex and model is None:
+                model = _lazy_model()
+            result = run(pdf, out, latex=latex, options=options, equation_model=model,
+                         progress=lambda message: print(f"    {message}", flush=True))
+        except Exception as error:  # one bad PDF should not stop the batch
+            failed += 1
+            print(f"    failed: {error}", file=sys.stderr)
+            continue
+        for note in result.doc.notes:
+            print(f"    - {note}")
+        print(f"    done in {time.time() - start:.0f} s: {result.markdown}")
+    return 1 if failed else 0
+
+
+def _lazy_model():
+    """Load pix2tex only when the first equation needs it."""
+    from pdfdown.equations import Pix2Tex
+
+    class Lazy:
+        name = "pix2tex"
+        _model = None
+
+        def __call__(self, image):
+            if self._model is None:
+                self._model = Pix2Tex()
+            return self._model(image)
+
+    return Lazy()
+
+
+def _check(argv: list[str]) -> int:
+    from pdfdown.quality import scan
+
+    if not argv:
+        print("usage: pdfdown check FILE.md ...", file=sys.stderr)
+        return 2
+    bad = 0
+    for name in argv:
+        findings = scan(Path(name))
+        print(f"{name}: {'no problems found' if not findings else ''}")
+        for finding in findings:
+            print(f"  - {finding}")
+        bad += bool(findings)
+    return 1 if bad else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())