agentpdf 0.1.0__py3-none-any.whl

agentpdf/__init__.py

@@ -0,0 +1,13 @@
+"""Convert large PDF documents into small, searchable files for AI coding agents."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("agentpdf")
+except PackageNotFoundError:
+    __version__ = "0.1.0"
+
+from .convert import Options, convert
+from .search import Hit, build_index, search
+
+__all__ = ["Hit", "Options", "__version__", "build_index", "convert", "search"]

agentpdf/chars.py

@@ -0,0 +1,265 @@
+"""Characters from PDFium, grouped into lines, words and layout text.
+
+PDFium follows the content stream order (usually the intended reading order,
+including multi-column layouts), reconstructs word spacing and decodes glyphs
+reliably.
+"""
+
+import ctypes
+import itertools
+import math
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import pypdfium2 as pdfium
+import pypdfium2.raw as pdfium_c
+
+from .model import BBox, Line, bbox_union
+
+# Characters PDFium uses to mark a hyphen at a line break.
+_SOFT_HYPHENS = "\ufffe\xad\u0002"
+
+
+@dataclass(slots=True)
+class Char:
+    text: str
+    index: int  # position in PDFium's text stream
+    bbox: BBox | None  # None for whitespace and line breaks
+    size: float = 0.0
+    bold: bool = False
+    upright: bool = True
+    soft_hyphen: bool = False
+
+
+Word = list[Char]
+
+
+def _display_transform(page: pdfium.PdfPage) -> Callable[[float, float, float, float], BBox]:
+    """Map PDF-space boxes (left, bottom, right, top) to displayed page
+    coordinates (top-left origin, after /Rotate)."""
+    mx0, my0, mx1, my1 = page.get_mediabox()
+    width, height = mx1 - mx0, my1 - my0
+    rotation = page.get_rotation() % 360
+
+    def point(x: float, y: float) -> tuple[float, float]:
+        x, y = x - mx0, y - my0
+        if rotation == 90:
+            return y, x
+        if rotation == 180:
+            return width - x, y
+        if rotation == 270:
+            return height - y, width - x
+        return x, height - y
+
+    def transform(left: float, bottom: float, right: float, top: float) -> BBox:
+        ax, ay = point(left, bottom)
+        bx, by = point(right, top)
+        return (min(ax, bx), min(ay, by), max(ax, bx), max(ay, by))
+
+    return transform
+
+
+def read_chars(page: pdfium.PdfPage) -> list[Char]:
+    """Read all characters of a page in stream order."""
+    textpage = page.get_textpage()
+    try:
+        return _read_chars(page, textpage)
+    finally:
+        textpage.close()
+
+
+def _read_chars(page: pdfium.PdfPage, textpage: pdfium.PdfTextPage) -> list[Char]:
+    to_display = _display_transform(page)
+    rotation = math.radians(page.get_rotation())
+    raw = textpage.raw
+    font_buf = ctypes.create_string_buffer(256)
+    font_flags = ctypes.c_int()
+    matrix = pdfium_c.FS_MATRIX()
+    chars: list[Char] = []
+    last_style = None
+    font = ""
+    # Characters are read by index (not by slicing get_text_range(), whose
+    # string offsets can drift from character indices).
+    for i in range(textpage.count_chars()):
+        code = pdfium_c.FPDFText_GetUnicode(raw, i)
+        ch = chr(code) if 0 < code < 0x110000 and not 0xD800 <= code < 0xE000 else "�"
+        if ch == "\r":
+            continue
+        if ch in _SOFT_HYPHENS:
+            chars.append(Char("-", i, None, soft_hyphen=True))
+            continue
+        if ch.isspace() or pdfium_c.FPDFText_IsGenerated(raw, i) == 1:
+            chars.append(Char("\n" if ch == "\n" else " ", i, None))
+            continue
+        # The nominal font size is often 1 with the actual size applied via
+        # the text matrix; the effective size accounts for both.
+        pdfium_c.FPDFText_GetMatrix(raw, i, ctypes.byref(matrix))
+        size = round(pdfium_c.FPDFText_GetFontSize(raw, i) * (math.hypot(matrix.c, matrix.d) or 1.0), 2)
+        weight = pdfium_c.FPDFText_GetFontWeight(raw, i)
+        if (size, weight) != last_style:
+            pdfium_c.FPDFText_GetFontInfo(raw, i, font_buf, len(font_buf), ctypes.byref(font_flags))
+            font = font_buf.value.decode("utf-8", "replace")
+            last_style = (size, weight)
+        # Upright as displayed, i.e. taking the page's /Rotate into account.
+        angle = pdfium_c.FPDFText_GetCharAngle(raw, i)
+        displayed = (angle + rotation) % (2 * math.pi)
+        chars.append(
+            Char(
+                text=ch,
+                index=i,
+                bbox=to_display(*textpage.get_charbox(i, loose=True)),
+                size=size,
+                bold=weight >= 600 or "bold" in font.lower(),
+                upright=angle < 0 or min(displayed, 2 * math.pi - displayed) < 0.05,
+            )
+        )
+    return chars
+
+
+def lines_from_chars(chars: list[Char]) -> list[Line]:
+    """Group characters into lines at PDFium's line breaks, and where the
+    geometry shows a break PDFium did not report (e.g. on rotated pages)."""
+    lines: list[Line] = []
+    current: list[Char] = []
+
+    def flush() -> None:
+        visible = [c for c in current if c.bbox is not None]
+        text = " ".join("".join(c.text for c in current).split())
+        if visible and text:
+            lines.append(
+                Line(
+                    text=text,
+                    bbox=_union(visible),
+                    size=max(c.size for c in visible),
+                    bold=sum(c.bold for c in visible) > len(visible) / 2,
+                    order=visible[0].index,
+                    soft_hyphen=current[-1].soft_hyphen,
+                    tabbed=any(
+                        b.bbox[0] - a.bbox[2] > 1.5 * max(a.size, b.size)
+                        for a, b in itertools.pairwise(visible)
+                    ),
+                )
+            )
+        current.clear()
+
+    prev: Char | None = None
+    for c in chars:
+        if c.text == "\n":
+            flush()
+            prev = None
+            continue
+        if c.bbox is not None:
+            if prev is not None and current and _starts_new_line(prev, c):
+                flush()
+            prev = c
+        current.append(c)
+        if c.soft_hyphen:
+            flush()
+    flush()
+    return lines
+
+
+def _starts_new_line(prev: Char, c: Char) -> bool:
+    if prev.upright != c.upright:
+        return True
+    if not c.upright:
+        return False
+    no_vertical_overlap = c.bbox[1] >= prev.bbox[3] or c.bbox[3] <= prev.bbox[1]
+    moved_back = c.bbox[0] < prev.bbox[0] - max(prev.size, c.size)
+    return no_vertical_overlap or moved_back
+
+
+def words(chars: list[Char]) -> list[Word]:
+    """Split visible characters into words at whitespace."""
+    return [
+        list(group) for visible, group in itertools.groupby(chars, lambda c: c.bbox is not None) if visible
+    ]
+
+
+def word_bbox(word: Word) -> BBox:
+    return _union(word)
+
+
+def _union(chars: list[Char]) -> BBox:
+    bbox = chars[0].bbox
+    for c in chars[1:]:
+        bbox = bbox_union(bbox, c.bbox)
+    return bbox
+
+
+def layout_text(chars: list[Char], bbox: BBox) -> str:
+    """Render characters as plain text that preserves horizontal positions.
+
+    Upright characters are placed on a character grid; rows are formed from
+    vertically overlapping characters so sub- and superscripts stay on their
+    base line. Rotated text (axis labels etc.) is listed separately.
+    """
+    upright = [c for c in chars if c.bbox is not None and c.upright]
+    rotated = [c for c in chars if c.bbox is not None and not c.upright]
+    out = _grid_rows(upright, bbox) if upright else []
+    if rotated:
+        out += ["", "[rotated text] " + " | ".join(_runs(rotated))]
+    indent = min((len(s) - len(s.lstrip()) for s in out if s.strip()), default=0)
+    return "\n".join(s[indent:] for s in out).strip("\n")
+
+
+def _grid_rows(chars: list[Char], bbox: BBox) -> list[str]:
+    sizes = sorted(c.size for c in chars)
+    body = sizes[len(sizes) // 2] or 8.0
+    unit = body * 0.5  # approximate width of one monospace column
+
+    # Tallest glyphs define the rows; smaller ones (subscripts) join a row
+    # they overlap substantially.
+    rows: list[tuple[float, float, list[Char]]] = []
+    for c in sorted(chars, key=lambda c: -c.size):
+        top, bottom = c.bbox[1], c.bbox[3]
+        for row_top, row_bottom, members in rows:
+            overlap = min(bottom, row_bottom) - max(top, row_top)
+            if overlap > 0.4 * min(bottom - top, row_bottom - row_top):
+                members.append(c)
+                break
+        else:
+            rows.append((top, bottom, [c]))
+    rows.sort(key=lambda r: r[0])
+
+    out: list[str] = []
+    prev_bottom = None
+    for top, bottom, members in rows:
+        if prev_bottom is not None and top - prev_bottom > body * 1.2:
+            out.append("")
+        prev_bottom = bottom
+        members.sort(key=lambda c: c.bbox[0])
+        line: list[str] = []
+        prev: Char | None = None
+        for c in members:
+            if prev is not None and c.index == prev.index + 1 and c.bbox[0] - prev.bbox[2] < c.size * 0.2:
+                # Continuation of a word: glyph widths vary, so append
+                # directly instead of snapping to the grid.
+                line.append(c.text)
+            else:
+                col = max(round((c.bbox[0] - bbox[0]) / unit), len(line) + (1 if line else 0))
+                line.extend(" " * (col - len(line)))
+                line.append(c.text)
+            prev = c
+        out.append("".join(line).rstrip())
+    return out
+
+
+def _runs(chars: list[Char]) -> list[str]:
+    """Join characters that are consecutive in the stream into labels."""
+    labels: list[str] = []
+    current: list[str] = []
+    last_index = 0
+    for c in sorted(chars, key=lambda c: c.index):
+        if current:
+            step = c.index - last_index
+            if step > 2:
+                labels.append("".join(current).strip())
+                current = []
+            elif step == 2:  # separated by a space
+                current.append(" ")
+        current.append(c.text)
+        last_index = c.index
+    if current:
+        labels.append("".join(current).strip())
+    return labels

agentpdf/cli.py

@@ -0,0 +1,98 @@
+"""Command-line interface."""
+
+import argparse
+import sys
+import time
+from pathlib import Path
+
+from . import __version__
+from .convert import Options, convert
+from .search import INDEX_FILE, build_index, format_hits, search
+from .writers import write_agents_md
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="agentpdf",
+        description="Convert PDF documents into small, searchable files for AI coding agents.",
+    )
+    parser.add_argument("--version", action="version", version=f"agentpdf {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    conv = sub.add_parser("convert", help="convert one or more PDF files")
+    conv.add_argument("pdf", nargs="+", help="PDF files to convert")
+    conv.add_argument("-o", "--output", required=True, help="output directory (one subdirectory per PDF)")
+    conv.add_argument("--dpi", type=int, default=150, help="resolution of rendered images (default: 150)")
+    conv.add_argument(
+        "--max-tokens",
+        type=int,
+        default=6000,
+        help="approximate size limit of a section file in tokens; larger sections are split at "
+        "subheadings, or into parts (default: 6000)",
+    )
+    conv.add_argument("--page-images", action="store_true", help="also render every full page as PNG")
+    conv.add_argument("--split-pdf", action="store_true", help="also write one PDF per section")
+    conv.add_argument("--no-index", action="store_true", help="do not build the search index")
+    conv.add_argument(
+        "--force", action="store_true", help="write into an existing non-empty output directory"
+    )
+
+    find = sub.add_parser("search", help="search converted documents")
+    find.add_argument("query", help='search terms; "quoted phrase", prefix*')
+    find.add_argument(
+        "-d", "--dir", default=".", help="output directory of `convert` (default: current directory)"
+    )
+    find.add_argument("-n", "--limit", type=int, default=10, help="maximum number of results (default: 10)")
+    find.add_argument("--doc", help="restrict to one document (its directory name)")
+    find.add_argument("--any", action="store_true", help="match any term instead of all terms")
+
+    args = parser.parse_args(argv)
+    if args.command == "search":
+        return _search(args)
+    return _convert(args)
+
+
+def _convert(args: argparse.Namespace) -> int:
+    options = Options(
+        dpi=args.dpi,
+        max_tokens=args.max_tokens,
+        page_images=args.page_images,
+        split_pdf=args.split_pdf,
+        index=False,  # built once below
+        force=args.force,
+    )
+    status = 0
+    converted = 0
+    for path in args.pdf:
+        start = time.monotonic()
+        try:
+            out_dir = convert(path, args.output, options)
+        except Exception as exc:
+            print(f"error: {path}: {exc}", file=sys.stderr)
+            status = 1
+            continue
+        converted += 1
+        print(f"{path} -> {out_dir} ({time.monotonic() - start:.1f}s)")
+    if converted and not args.no_index:
+        build_index(Path(args.output))
+        write_agents_md(Path(args.output), has_search=(Path(args.output) / INDEX_FILE).is_file())
+    return status
+
+
+def _search(args: argparse.Namespace) -> int:
+    try:
+        hits, relaxed = search(Path(args.dir), args.query, args.limit, args.doc, args.any)
+    except FileNotFoundError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    if not hits:
+        print("no matches", file=sys.stderr)
+        return 1
+    if relaxed:
+        print("(no entry contains all terms; showing entries matching any term)")
+    print(format_hits(hits))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agentpdf/convert.py

@@ -0,0 +1,205 @@
+"""Convert a PDF into a directory of Markdown sections, images and indexes."""
+
+import hashlib
+import json
+import shutil
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+import pypdf
+
+from .document import Document
+from .furniture import drop_printed_toc, remove_furniture
+from .headings import detect_headings, mark_outline_headings, repair_outline_levels
+from .paragraphs import Block, page_blocks, resolve_hyphens
+from .search import INDEX_FILE, build_index
+from .sections import Section, build_sections
+from .util import page_range, slugify
+from .writers import (
+    MANIFEST,
+    MARKER,
+    is_agentpdf_agents_md,
+    is_agentpdf_dir,
+    manifest,
+    pages_markdown,
+    section_markdown,
+    write_agents_md,
+    write_index,
+)
+
+
+@dataclass
+class Options:
+    dpi: int = 150  # resolution of rendered images
+    max_tokens: int = 6000  # approximate size limit of a section file
+    page_images: bool = False  # also render every full page
+    split_pdf: bool = False  # also write one PDF per section
+    index: bool = True  # rebuild the search index after converting
+    force: bool = False  # allow existing non-empty output root
+
+
+def _prepare_out_root(out_root: Path, force: bool) -> None:
+    if out_root.is_symlink() or (out_root.exists() and not out_root.is_dir()):
+        raise FileExistsError(f"{out_root} exists and is not a directory")
+    if out_root.exists():
+        has_marker = (out_root / MARKER).is_file() and not (out_root / MARKER).is_symlink()
+        if not has_marker and any(out_root.iterdir()) and not force:
+            raise FileExistsError(
+                f"{out_root} already exists and was not created by agentpdf (use --force to use it anyway)"
+            )
+        agents_path = out_root / "AGENTS.md"
+        if (agents_path.exists() or agents_path.is_symlink()) and not is_agentpdf_agents_md(agents_path):
+            raise FileExistsError(f"{agents_path} exists and was not created by agentpdf")
+        index_path = out_root / INDEX_FILE
+        if index_path.is_symlink() or index_path.is_dir() or (not has_marker and index_path.exists()):
+            raise FileExistsError(f"{index_path} exists and was not created by agentpdf")
+    else:
+        out_root.mkdir(parents=True)
+
+
+def _swap_dir(work_dir: Path, final_dir: Path) -> None:
+    if not final_dir.exists():
+        work_dir.rename(final_dir)
+        return
+    backup = tempfile.mkdtemp(prefix=f".{final_dir.name}.", suffix=".backup", dir=final_dir.parent)
+    backup_dir = Path(backup)
+    backup_dir.rmdir()
+    final_dir.rename(backup_dir)
+    try:
+        work_dir.rename(final_dir)
+    except BaseException:
+        backup_dir.rename(final_dir)
+        shutil.rmtree(work_dir, ignore_errors=True)
+        raise
+    shutil.rmtree(backup_dir, ignore_errors=True)
+
+
+def _manifest_sha256(doc_dir: Path) -> str | None:
+    try:
+        info = json.loads((doc_dir / MANIFEST).read_text(encoding="utf-8"))
+        return info["source"]["sha256"]
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+
+
+def _target_dir(out_root: Path, slug: str, sha256: str) -> Path:
+    n = 1
+    while True:
+        name = slug if n == 1 else f"{slug}-{n}"
+        candidate = out_root / name
+        if not candidate.exists() and not candidate.is_symlink():
+            return candidate
+        if not is_agentpdf_dir(candidate):
+            raise FileExistsError(f"{candidate} exists and was not created by agentpdf")
+        if _manifest_sha256(candidate) == sha256:
+            return candidate
+        n += 1
+
+
+def convert(pdf_path: str | Path, out_root: str | Path, options: Options | None = None) -> Path:
+    """Convert one PDF into ``out_root/<name>/`` and update the shared
+    AGENTS.md and search index in ``out_root``. Returns the document's
+    output directory.
+
+    The output is built in a temporary directory and swapped in when
+    complete, so an interrupted conversion leaves the previous result intact.
+    """
+    options = options or Options()
+    source, out_root = Path(pdf_path), Path(out_root)
+    sha256 = _sha256(source)
+    _prepare_out_root(out_root, options.force)
+    final_dir = _target_dir(out_root, slugify(source.stem), sha256)
+
+    work_dir = Path(tempfile.mkdtemp(prefix=f".{final_dir.name}.", suffix=".partial", dir=out_root))
+    try:
+        _convert(source, sha256, work_dir, options)
+    except BaseException:
+        shutil.rmtree(work_dir, ignore_errors=True)
+        raise
+    _swap_dir(work_dir, final_dir)
+    marker = out_root / MARKER
+    if not marker.exists():
+        marker.write_text("agentpdf\n", encoding="utf-8")
+
+    if options.index:
+        build_index(out_root)
+    write_agents_md(out_root, has_search=(out_root / INDEX_FILE).is_file())
+    return final_dir
+
+
+def _convert(source: Path, sha256: str, out_dir: Path, options: Options) -> None:
+    (out_dir / "sections").mkdir()
+    (out_dir / "images").mkdir()
+
+    with Document(str(source)) as doc:
+        pages = list(doc.pages())
+        outline = doc.outline()
+        remove_furniture(pages)
+        toc_pages = drop_printed_toc(pages, outline)
+
+        # Tables and figures become images; pages without text are rendered whole.
+        for page in pages:
+            for region in page.regions:
+                region.image = f"images/p{page.number:04d}-{region.id}.png"
+                doc.render(page.number - 1, str(out_dir / region.image), options.dpi, region.bbox)
+            if options.page_images or not page.has_text_layer:
+                page.image = f"images/p{page.number:04d}.png"
+                doc.render(page.number - 1, str(out_dir / page.image), options.dpi)
+
+        blocks = [b for page in pages for b in page_blocks(page)]
+        blocks += [
+            Block(p.number, -1, f"*(This page has no text layer. See [page image](../{p.image}).)*")
+            for p in pages
+            if not p.has_text_layer
+        ]
+        if toc_pages:
+            pages_text = page_range((toc_pages[0], toc_pages[-1]))
+            note = f"*(Printed table of contents omitted ({pages_text}); see [INDEX.md](../INDEX.md).)*"
+            blocks.append(Block(toc_pages[0], -1, note))
+        blocks.sort(key=lambda b: (b.page, b.order))
+        resolve_hyphens(blocks)
+
+        if outline:
+            blocks = mark_outline_headings(blocks, repair_outline_levels(outline))
+            outline_source = "bookmarks"
+        else:
+            outline_source = "typography" if detect_headings(blocks) else "none"
+        sections = build_sections(blocks, options.max_tokens * 4)
+
+        for section in sections:
+            if section.has_file:
+                (out_dir / "sections" / section.filename).write_text(
+                    section_markdown(section), encoding="utf-8"
+                )
+        if options.split_pdf:
+            _split_pdf(source, sections, out_dir / "pdf")
+
+        info = manifest(source, sha256, doc.metadata, pages, sections, outline_source)
+        (out_dir / MANIFEST).write_text(
+            json.dumps(info, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
+        )
+        write_index(out_dir, info, sections)
+        (out_dir / "PAGES.md").write_text(pages_markdown(pages, sections), encoding="utf-8")
+
+
+def _split_pdf(source: Path, sections: list[Section], out_dir: Path) -> None:
+    out_dir.mkdir()
+    reader = pypdf.PdfReader(source)
+    for section in sections:
+        if not section.has_file or section.pages is None:
+            continue
+        first, last = section.pages
+        writer = pypdf.PdfWriter()
+        for number in range(first, last + 1):
+            writer.add_page(reader.pages[number - 1])
+        with open(out_dir / section.filename.replace(".md", ".pdf"), "wb") as f:
+            writer.write(f)
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()