backdraft 0.2.0__py3-none-any.whl

backdraft/extract/base.py

@@ -0,0 +1,184 @@
+"""The Extractor protocol and the registry of extractors.
+
+An extractor turns a file into an ordered sequence of `ExtractedPage`s — a PDF
+page, a sheet, or the single page a text file is. That sequence is the snapshot
+the registry stores and every receipt quotes; nothing downstream reopens the
+file.
+
+`deterministic` is the honest self-report that makes the re-ingest contract
+possible: a deterministic extractor run twice over identical bytes produces
+identical pages, so the registry can skip the work and keep every token. The VLM
+extractor says `False`; it is the preferred `auto` choice for PDFs when it is
+ready (installed, key configured), because the snapshot is the receipt and
+glossy layouts extract badly from the text layer.
+
+Extractors are registered in a plain dict keyed by name. Modules register
+themselves on import and `get` imports the module that owns a name on demand, so
+neither pdfplumber nor openpyxl — nor the `[vlm]` extra — is imported until an
+extractor that needs it is asked for.
+"""
+
+from __future__ import annotations
+
+import importlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, Protocol, runtime_checkable
+
+from ..kernel.errors import BackdraftError
+from ..kernel.model import CellValue, PageKind
+
+__all__ = [
+    "AUTO_ORDER",
+    "vlm_ready",
+    "EXTRACTORS",
+    "ExtractedPage",
+    "Extractor",
+    "ExtractionError",
+    "PageImage",
+    "get",
+    "names",
+    "register",
+    "select",
+]
+
+
+class ExtractionError(BackdraftError):
+    """An extractor could not produce a snapshot of a file."""
+
+
+@dataclass(frozen=True, slots=True)
+class PageImage:
+    """A page's visual snapshot: the image an extractor derived the text from.
+
+    For the VLM extractor this is the page as the model was shown it (re-encoded
+    WebP, quality 85 — the embed-ses production shape), which for a
+    non-deterministic extractor is the one reproducible half of the receipt.
+    """
+
+    data: bytes
+    format: str  # 'webp' | 'png' | 'jpeg'
+    width: int
+    height: int
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractedPage:
+    """One ordered unit of a snapshot: a PDF page or a sheet.
+
+    `text` is the page as the model will read it — for sheets, the markdown table
+    with in-band `[B10]` cell references. `cells` carries the sheet's populated
+    cells so the registry can mint one anchor per cell and value-trace has values
+    to compare against; each `value` is a verbatim substring of `text`. `image`,
+    when present, is the page's visual snapshot; the registry stores it so bind
+    can embed the cited pages into the artifact.
+    """
+
+    number: int
+    kind: PageKind
+    text: str
+    name: str | None = None
+    cells: list[CellValue] | None = None
+    image: PageImage | None = None
+
+
+@runtime_checkable
+class Extractor(Protocol):
+    """Turns a file into pages. Stateless: one instance serves every ingest."""
+
+    name: str
+    version: str
+    deterministic: bool
+
+    def can_handle(self, path: Path, media_type: str) -> bool:
+        """True if this extractor claims the file."""
+        ...
+
+    def extract(self, path: Path, config: dict) -> Iterator[ExtractedPage]:
+        """Yield the file's pages in order, 1-based. Raises `ExtractionError`."""
+        ...
+
+
+EXTRACTORS: dict[str, Extractor] = {}
+"""Name -> extractor. Populated as modules are imported."""
+
+AUTO_ORDER = ("xlsx", "pdf-text", "text")
+"""The built-in fallback order for `--extractor auto`. `vlm` is preferred
+for PDFs when ready — see `select` — and otherwise only chosen by name."""
+
+# NOTE: the spec is silent on how a name finds its module; a table beats a scan
+# because it keeps the import of an extractor's dependencies lazy.
+_MODULES = {
+    "xlsx": "xlsx",
+    "pdf-text": "pdf_text",
+    "text": "text",
+    "vlm": "vlm",
+}
+
+
+def register(extractor: Extractor) -> None:
+    """Register an extractor under its own name."""
+    EXTRACTORS[extractor.name] = extractor
+
+
+def names() -> list[str]:
+    """Every extractor name this build knows, whether or not it can be loaded."""
+    return sorted(_MODULES)
+
+
+def get(name: str) -> Extractor:
+    """The extractor called `name`, importing its module on first use.
+
+    Raises `ExtractionError` for an unknown name, or when the module cannot be
+    imported — the `[vlm]` extra not being installed is the case that matters.
+    """
+    if name in EXTRACTORS:
+        return EXTRACTORS[name]
+    if name not in _MODULES:
+        raise ExtractionError(f"unknown extractor {name!r}; known: {', '.join(names())}")
+    try:
+        importlib.import_module(f"{__package__}.{_MODULES[name]}")
+    except ImportError as error:
+        raise ExtractionError(f"extractor {name!r} is unavailable: {error}") from error
+    if name not in EXTRACTORS:  # pragma: no cover - a module that forgot to register
+        raise ExtractionError(f"extractor module for {name!r} registered nothing")
+    return EXTRACTORS[name]
+
+
+def select(path: Path, media_type: str, config: dict | None = None) -> Extractor:
+    """The extractor `--extractor auto` picks.
+
+    For PDFs, `vlm` is preferred when it is ready (installed, key configured):
+    glossy layouts and info boxes extract badly from the text layer, and the
+    snapshot is the receipt. Otherwise the first built-in that can handle the
+    file wins; the CLI owns the one-line nudge naming the VLM option, so
+    selection itself stays quiet. An explicit `--extractor` never reaches this
+    function.
+    """
+    if media_type == "pdf" and vlm_ready(config):
+        extractor = get("vlm")
+        if extractor.can_handle(path, media_type):
+            return extractor
+    for name in AUTO_ORDER:
+        extractor = get(name)
+        if extractor.can_handle(path, media_type):
+            return extractor
+    raise ExtractionError(f"no extractor handles {media_type!r} file {path.name!r}")
+
+
+def vlm_ready(config: dict | None = None) -> bool:
+    """True when the `[vlm]` extra is importable and a backdraft-scoped key exists.
+
+    Equivalent to consent: `auto` may only choose the paid, off-machine path
+    when the user deliberately configured backdraft to use it. Ambient
+    provider keys are never read (see `backdraft.credentials`).
+    """
+    from ..credentials import setting  # noqa: PLC0415 - keep base import-light
+
+    if not setting("BACKDRAFT_VLM_API_KEY", config, config_key="api_key"):
+        return False
+    try:
+        get("vlm")
+    except ExtractionError:
+        return False
+    return True

backdraft/extract/pdf_text.py

@@ -0,0 +1,124 @@
+"""PDF text layer, via pdfplumber. One `ExtractedPage` per PDF page.
+
+This extractor reads the text the PDF already carries; it does not look at the
+pixels. A scan has no text layer, so it produces nothing to anchor — that is an
+error naming the `[vlm]` extra rather than an empty registry entry, because a
+document that silently ingests as blank is worse than one that refuses.
+
+**Paragraph reconstruction.** `page.extract_text()` joins every line with a
+single `\\n` and never emits a blank one, because a PDF has no paragraphs — it
+has glyphs at coordinates. The chunker's first rule splits on blank lines, so
+without this step that rule is dead on every PDF and sub-page anchors only ever
+appear via the long-segment split. The paragraph break is still *in* the file,
+just expressed as vertical space, so this extractor reads it back off the
+geometry: lines in document order, the page's median line pitch as the body
+leading, and a blank line inserted wherever the pitch jumps past
+`PARAGRAPH_GAP_RATIO` times that median.
+
+The line texts themselves are pdfplumber's own — `"\\n".join(extract_text_lines())`
+is character-for-character `extract_text()` — so this only ever *adds* blank
+lines. NOTE: it is a heuristic over a format that does not carry the answer.
+A layout it reads wrong shifts chunk boundaries; it does so deterministically,
+which is the property anchors actually depend on.
+"""
+
+from __future__ import annotations
+
+import statistics
+from pathlib import Path
+from typing import Any, Iterator
+
+import pdfplumber
+
+from .base import ExtractedPage, Extractor, ExtractionError, register
+
+__all__ = ["PARAGRAPH_GAP_RATIO", "PdfTextExtractor", "EXTRACTOR", "page_text"]
+
+PARAGRAPH_GAP_RATIO = 1.5
+"""A line starting this much further down than the median pitch opens a paragraph.
+
+Tuned on the demo corpus, whose gaps fall in three tight clusters: 14.0pt
+between body lines, 24.0pt between paragraphs (1.71x), and 31.1pt below a
+heading (2.22x). 1.5 sits in the empty band between the first two clusters,
+which is where typographic convention puts it too — space-after under half the
+leading reads as one block, over half reads as a break.
+"""
+
+_NO_TEXT_LAYER = (
+    "{path} has no text layer on any page — it is probably a scan. "
+    "Install the VLM extractor and pass it explicitly: "
+    "`pip install 'backdraft[vlm]'` then `backdraft ingest {path} --extractor vlm`."
+)
+
+
+class PdfTextExtractor:
+    """pdfplumber text-layer extraction. Deterministic for a given pdfplumber."""
+
+    name = "pdf-text"
+    version = "1"
+    deterministic = True
+
+    def can_handle(self, path: Path, media_type: str) -> bool:
+        return media_type == "pdf"
+
+    def extract(self, path: Path, config: dict) -> Iterator[ExtractedPage]:
+        """Yield one page per PDF page, in file order.
+
+        Individual blank pages are yielded as empty pages — they are part of the
+        document and their numbers must not shift. Only a PDF with no text
+        anywhere raises.
+        """
+        yield from self._pages(path)
+
+    def _pages(self, path: Path) -> Iterator[ExtractedPage]:
+        found_text = False
+        try:
+            document = pdfplumber.open(path)
+        except Exception as error:  # noqa: BLE001 - pdfminer raises broadly
+            raise ExtractionError(f"could not open {path}: {error}") from error
+        with document:
+            for number, page in enumerate(document.pages, start=1):
+                text = page_text(page)
+                if text.strip():
+                    found_text = True
+                yield ExtractedPage(number=number, kind="page", text=text)
+        if not found_text:
+            raise ExtractionError(_NO_TEXT_LAYER.format(path=path))
+
+
+def page_text(page: Any) -> str:
+    """One page's text, with paragraph breaks restored from its layout.
+
+    Falls back to `extract_text()` whenever the geometry cannot answer: no
+    lines, a single line, or a degenerate median pitch. The fallback is the
+    old behaviour exactly, so a PDF this cannot read is no worse off.
+    """
+    try:
+        lines = page.extract_text_lines()
+    except Exception:  # noqa: BLE001 - pdfminer raises broadly; never lose the page
+        lines = []
+    if len(lines) < 2:
+        return page.extract_text() or ""
+    return _join(lines)
+
+
+def _join(lines: list[dict[str, Any]]) -> str:
+    """Line texts joined, blank-line separated wherever the pitch jumps."""
+    gaps = [
+        float(later["top"]) - float(earlier["top"])
+        for earlier, later in zip(lines, lines[1:], strict=False)
+    ]
+    pitch = statistics.median(gaps)
+    if pitch <= 0:
+        # Overlapping or unordered lines: no baseline to compare against.
+        return "\n".join(line["text"] for line in lines)
+    threshold = pitch * PARAGRAPH_GAP_RATIO
+    out = [lines[0]["text"]]
+    for gap, line in zip(gaps, lines[1:], strict=True):
+        out.append("\n\n" if gap > threshold else "\n")
+        out.append(line["text"])
+    return "".join(out)
+
+
+EXTRACTOR: Extractor = PdfTextExtractor()
+register(EXTRACTOR)

backdraft/extract/text.py

@@ -0,0 +1,44 @@
+"""Markdown and plain text: the whole file is one page.
+
+A text file has no native pagination, so it gets page 1 and the chunker supplies
+the sub-page structure — `bd:notes:p1.c3:a7f3` addresses the third paragraph
+group. NOTE: the spec leaves heading-path and line-range locator forms open; page
+1 plus chunks needs no new locator form, so v0 uses it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator
+
+from .base import ExtractedPage, Extractor, register
+
+__all__ = ["TextExtractor", "EXTRACTOR"]
+
+
+class TextExtractor:
+    """md/txt passthrough. Deterministic: it is a decode."""
+
+    name = "text"
+    version = "1"
+    deterministic = True
+
+    def can_handle(self, path: Path, media_type: str) -> bool:
+        return media_type == "text"
+
+    def extract(self, path: Path, config: dict) -> Iterator[ExtractedPage]:
+        """The file's text as page 1.
+
+        NOTE: undecodable bytes are replaced rather than raising. A snapshot with
+        a replacement character still anchors; a failed ingest anchors nothing.
+        Universal newlines mean CRLF and LF files snapshot identically.
+        """
+        yield ExtractedPage(
+            number=1,
+            kind="page",
+            text=path.read_text(encoding="utf-8", errors="replace"),
+        )
+
+
+EXTRACTOR: Extractor = TextExtractor()
+register(EXTRACTOR)

backdraft/extract/vlm.py

@@ -0,0 +1,175 @@
+"""The `[vlm]` extra: pdf -> page images -> a vision model, one page at a time.
+
+The zerox-minimal pattern, kept thin: render each page, ask an OpenAI-compatible
+vision model for markdown, use the answer as the page. This is the recommended
+path for real PDFs — glossy layouts, info boxes, scans — because the snapshot is
+the receipt, and the model's clean reading of the page beats scrambled
+text-layer order.
+
+The client is always the `openai` SDK; a router is injected through `base_url`.
+The default provider is OpenRouter running Gemini 3.1 Flash Lite (what the
+HighRoad pipeline uses in production); a bare `OPENAI_API_KEY` routes direct to
+OpenAI instead. See `client_settings`.
+
+`deterministic = False`, honestly: the same PDF ingested twice yields different
+text, so the registry never short-circuits a re-ingest and every anchor whose
+snippet moved gets a new token. That is the correct, visible outcome — a model's
+transcription is not a snapshot anyone can reproduce.
+
+`--extractor auto` prefers this extractor for PDFs when it is ready (installed,
+key configured); otherwise `auto` falls back to the text layer and the CLI says
+so. Importing this module requires the extra (`pip install 'backdraft[vlm]'`)
+and `pdf2image` requires poppler on the PATH.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import sys
+import tempfile
+from pathlib import Path
+from typing import Iterator
+
+from openai import OpenAI
+from pdf2image import convert_from_path
+
+from .base import ExtractedPage, Extractor, ExtractionError, PageImage, register
+from .vlm_settings import (  # noqa: F401  (re-exported)
+    DEFAULT_MODEL,
+    MAX_IMAGE_HEIGHT,
+    OPENROUTER_BASE_URL,
+    SNAPSHOT_QUALITY,
+    client_settings,
+    concurrency,
+    retries,
+    run_ordered,
+    timeout_seconds,
+    with_retries,
+)
+
+__all__ = [
+    "DEFAULT_MODEL",
+    "OPENROUTER_BASE_URL",
+    "SYSTEM_PROMPT",
+    "VlmExtractor",
+    "EXTRACTOR",
+    "client_settings",
+]
+
+DEFAULT_DPI = 200
+
+SYSTEM_PROMPT = (
+    "Convert this document page to markdown. Follow these rules strictly:\n\n"
+    "VERBATIM TEXT: Write all visible text exactly as shown — headings, labels, "
+    "captions, data values.\n\n"
+    "VISUAL ELEMENTS: Use [IMAGE: description], [MAP: description] or "
+    "[CHART: description] brackets. Describe what the visual communicates.\n\n"
+    "TABLES: Use markdown table format with all data preserved.\n\n"
+    "STRUCTURE: Use markdown headings (#, ##) to reflect page hierarchy.\n\n"
+    "DO NOT: add commentary, preambles like 'Here is...', or explanatory text "
+    "outside brackets. Never invent URLs.\n\n"
+    "Output only the converted content."
+)
+
+
+class VlmExtractor:
+    """pdf -> page images -> vision model. Non-deterministic by construction."""
+
+    name = "vlm"
+    version = "1"
+    deterministic = False
+
+    def can_handle(self, path: Path, media_type: str) -> bool:
+        return media_type == "pdf"
+
+    def extract(self, path: Path, config: dict) -> Iterator[ExtractedPage]:
+        """Yield one page per PDF page, transcribed by the model.
+
+        Config keys (all optional): `model`, `base_url`, `api_key`, `dpi`.
+        Provider resolution is `client_settings`; OpenRouter is first-class.
+        """
+        model, api_key, base_url = client_settings(config)
+        client = OpenAI(api_key=api_key, base_url=base_url, timeout=timeout_seconds(config))
+        dpi = int(config.get("dpi") or DEFAULT_DPI)
+        attempts = retries(config)
+
+        with tempfile.TemporaryDirectory() as workdir:
+            try:
+                images = convert_from_path(str(path), dpi=dpi, fmt="png")
+            except Exception as error:  # noqa: BLE001 - pdf2image raises broadly
+                raise ExtractionError(f"could not render {path} to images: {error}") from error
+            image_paths: list[Path] = []
+            snapshots: list[PageImage] = []
+            for number, image in enumerate(images, start=1):
+                if image.height > MAX_IMAGE_HEIGHT:
+                    scale = MAX_IMAGE_HEIGHT / image.height
+                    image = image.resize(
+                        (max(1, round(image.width * scale)), MAX_IMAGE_HEIGHT)
+                    )
+                image_path = Path(workdir) / f"page_{number:04d}.png"
+                image.save(image_path, format="PNG")
+                image_paths.append(image_path)
+                buffer = io.BytesIO()
+                image.save(buffer, format="WEBP", quality=SNAPSHOT_QUALITY)
+                snapshots.append(PageImage(
+                    data=buffer.getvalue(), format="webp",
+                    width=image.width, height=image.height,
+                ))
+
+            texts = run_ordered(
+                image_paths,
+                lambda image_path: with_retries(
+                    lambda: _transcribe(client, model, image_path), attempts=attempts
+                ),
+                workers=concurrency(config),
+                progress=_progress(path.name, len(image_paths)),
+            )
+            for number, text in enumerate(texts, start=1):
+                yield ExtractedPage(
+                    number=number, kind="page", text=text,
+                    image=snapshots[number - 1],
+                )
+
+
+def _progress(name: str, total: int):
+    """Progress to stderr: a live counter on a tty, one line per page otherwise."""
+    tty = sys.stderr.isatty()
+
+    def report(done: int, _total: int) -> None:
+        if tty:
+            end = "\n" if done == total else "\r"
+            print(f"vlm: {name}  {done}/{total} pages", file=sys.stderr, end=end, flush=True)
+        else:
+            print(f"vlm: {name}  page {done}/{total}", file=sys.stderr, flush=True)
+
+    return report
+
+
+def _transcribe(client: OpenAI, model: str, image_path: Path) -> str:
+    """One page image -> markdown. Raises `ExtractionError` on any API failure."""
+    encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")
+    try:
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Extract text from this page."},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{encoded}"},
+                        },
+                    ],
+                },
+            ],
+        )
+    except Exception as error:  # noqa: BLE001 - client raises broadly
+        raise ExtractionError(f"vlm call failed for {image_path.name}: {error}") from error
+    return response.choices[0].message.content or ""
+
+
+EXTRACTOR: Extractor = VlmExtractor()
+register(EXTRACTOR)