backdraft 0.2.0__py3-none-any.whl

backdraft/extract/vlm_settings.py

@@ -0,0 +1,197 @@
+"""Provider resolution for the VLM extractor — importable without the extra.
+
+The client is always the `openai` SDK; a router is injected through `base_url`,
+never through a second SDK. This module is stdlib-only so `client_settings` can
+be imported (and tested) without `backdraft[vlm]` installed; `vlm.py` re-exports
+it next to the extractor that uses it.
+"""
+
+from __future__ import annotations
+
+from ..credentials import setting
+from .base import ExtractionError
+
+__all__ = [
+    "DEFAULT_CONCURRENCY",
+    "DEFAULT_RETRIES",
+    "DEFAULT_TIMEOUT_SECONDS",
+    "MAX_IMAGE_HEIGHT",
+    "DEFAULT_MODEL",
+    "OPENROUTER_BASE_URL",
+    "client_settings",
+    "concurrency",
+    "is_retryable",
+    "retries",
+    "run_ordered",
+    "timeout_seconds",
+    "with_retries",
+]
+
+DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"
+"""The default: Gemini 3.1 Flash Lite through OpenRouter — what the HighRoad
+pipeline runs in production. Overridden with `--config model=...` or
+`BACKDRAFT_VLM_MODEL`."""
+
+OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
+
+
+def client_settings(config: dict) -> tuple[str, str, str | None]:
+    """Resolve `(model, api_key, base_url)` from backdraft-scoped settings only.
+
+    The client is always the `openai` SDK; a router is injected through
+    `base_url`, never through a second SDK. There is one default provider —
+    OpenRouter running Gemini 3.1 Flash Lite — and one rule for credentials:
+    **ambient provider variables are never read.** `OPENAI_API_KEY` and
+    `OPENROUTER_API_KEY` exported for other tools do not exist as far as this
+    function is concerned; presence of a generic key is not consent to send
+    documents to its provider. See `backdraft.credentials`.
+
+    * `api_key`: `--config api_key=` → `BACKDRAFT_VLM_API_KEY` (env or
+      `.backdraft/env`). Required.
+    * `base_url`: `--config base_url=` → `BACKDRAFT_VLM_BASE_URL` → OpenRouter.
+    * `model`: `--config model=` → `BACKDRAFT_VLM_MODEL` → Gemini 3.1 Flash
+      Lite. Direct OpenAI (or any OpenAI-compatible endpoint) is explicit:
+      set the base_url and the model together.
+    """
+    api_key = setting("BACKDRAFT_VLM_API_KEY", config, config_key="api_key")
+    if not api_key:
+        raise ExtractionError(
+            "the vlm extractor needs a backdraft-scoped API key: set "
+            "BACKDRAFT_VLM_API_KEY (env or .backdraft/env), or pass "
+            "--config api_key=... — ambient OPENAI_API_KEY/OPENROUTER_API_KEY "
+            "are deliberately not read"
+        )
+    base_url = (
+        setting("BACKDRAFT_VLM_BASE_URL", config, config_key="base_url")
+        or OPENROUTER_BASE_URL
+    )
+    model = setting("BACKDRAFT_VLM_MODEL", config, config_key="model") or DEFAULT_MODEL
+    return str(model), str(api_key), str(base_url)
+
+
+DEFAULT_CONCURRENCY = 15
+"""Concurrent page transcriptions — minimal_zerox's production value."""
+
+DEFAULT_TIMEOUT_SECONDS = 120
+"""Per-page request timeout, as tested in production."""
+
+DEFAULT_RETRIES = 4
+"""Attempts per page before the error propagates."""
+
+MAX_IMAGE_HEIGHT = 1056
+"""Rendered page images are downscaled to this height — token-cost control,
+ported from minimal_zerox (dpi buys legibility, height caps what a page costs)."""
+
+SNAPSHOT_QUALITY = 85
+"""WebP quality for stored page snapshots (embed-ses generate-preview's
+production setting). Lives here rather than vlm.py so `snapshot-pages` can
+import it without pulling in the openai client."""
+
+
+def _int_setting(name: str, config: dict, config_key: str, default: int) -> int:
+    raw = setting(name, config, config_key=config_key)
+    try:
+        return max(1, int(raw)) if raw else default
+    except ValueError:
+        return default
+
+
+def concurrency(config: dict) -> int:
+    """`--config concurrency=` → `BACKDRAFT_VLM_CONCURRENCY` → 15."""
+    return _int_setting("BACKDRAFT_VLM_CONCURRENCY", config, "concurrency", DEFAULT_CONCURRENCY)
+
+
+def timeout_seconds(config: dict) -> int:
+    """`--config timeout=` → `BACKDRAFT_VLM_TIMEOUT` → 120."""
+    return _int_setting("BACKDRAFT_VLM_TIMEOUT", config, "timeout", DEFAULT_TIMEOUT_SECONDS)
+
+
+def retries(config: dict) -> int:
+    """`--config retries=` → `BACKDRAFT_VLM_RETRIES` → 4."""
+    return _int_setting("BACKDRAFT_VLM_RETRIES", config, "retries", DEFAULT_RETRIES)
+
+
+def run_ordered(items, fn, *, workers, progress=None):
+    """Map `fn` over `items` concurrently, yielding results in input order.
+
+    `progress(done, total)` is called after each completion, in completion
+    order — the caller renders it however it likes. Exceptions propagate from
+    the yield of the item that raised, so ordering of failures is deterministic
+    too.
+    """
+    import threading
+    from concurrent.futures import ThreadPoolExecutor
+
+    items = list(items)
+    total = len(items)
+    done = 0
+    lock = threading.Lock()
+
+    def wrapped(item):
+        nonlocal done
+        try:
+            return fn(item)
+        finally:
+            with lock:
+                done += 1
+                if progress is not None:
+                    progress(done, total)
+
+    with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
+        futures = [pool.submit(wrapped, item) for item in items]
+        for future in futures:
+            yield future.result()
+
+
+_TRANSIENT_MARKERS = (
+    "rate limit",
+    "too many requests",
+    "overloaded",
+    "overload",
+    "temporarily unavailable",
+    "timed out",
+    "timeout",
+    "connection reset",
+    "connection aborted",
+    "gateway",
+    "upstream",
+    "server disconnected",
+)
+"""Message fragments that mark a transient upstream failure — minimal_zerox's
+production-tested classification, ported verbatim."""
+
+
+def is_retryable(error: Exception) -> bool:
+    """Whether a page transcription failure is worth another attempt.
+
+    Retryable: timeouts, HTTP 429/502/503/504 (via a `status_code` attribute,
+    the shape the openai SDK raises), transient upstream messages, and
+    connection-level failures. Everything else — auth, bad request, content —
+    fails fast; retrying those only spends money on the same answer.
+    """
+    if isinstance(error, TimeoutError):
+        return True
+    if getattr(error, "status_code", None) in {429, 502, 503, 504}:
+        return True
+    message = str(error).lower()
+    if any(marker in message for marker in _TRANSIENT_MARKERS):
+        return True
+    return isinstance(error, (ConnectionError, OSError))
+
+
+def with_retries(fn, *, attempts, base_delay=1.0, sleep=None, jitter=None):
+    """Run `fn` with minimal_zerox's retry shape: classify, exponential backoff
+    plus jitter, raise the last error once attempts are spent or on the first
+    non-retryable one. `sleep`/`jitter` are injectable for tests."""
+    import random
+    import time
+
+    sleep = sleep or time.sleep
+    jitter = jitter or (lambda high: random.uniform(0, high))
+    for attempt in range(1, max(1, attempts) + 1):
+        try:
+            return fn()
+        except Exception as error:  # noqa: BLE001 - classification decides
+            if not is_retryable(error) or attempt >= attempts:
+                raise
+            sleep(base_delay * (2 ** (attempt - 1)) + jitter(base_delay))

backdraft/extract/xlsx.py

@@ -0,0 +1,244 @@
+"""Workbooks: one page per sheet, rendered as a markdown table with in-band refs.
+
+This is the representation the HighRoad app proved — the only one whose sub-page
+attribution worked, because the coordinates sit *in the text the model reads*:
+
+```
+## Sheet: rent-roll - Values View with cell references
+
+| Row | A | B |
+|---|---|---|
+| 1 | [A1] Unit | [B1] Rent |
+| 2 | [A2] 101 | [B2] 2400 |
+```
+
+A model that copies `2400` out of that table has already seen `[B2]`, so the
+registry can compose `bd:model:rent-roll!B2:9e2f` for it. Nothing about the
+location travels in a side channel.
+
+The app's hardening is kept because every bit of it was earned on real files:
+hard row/column caps, an inflated-dimension placeholder page (Excel formats empty
+cells and reports a million rows), trailing empty row/column trimming, and cell
+truncation. NOTE: the caps mean a snapshot can be a *partial* view of a sheet;
+the page title says so, which is the honest failure the spec's "failures are
+data" principle asks for.
+
+Reader choice: openpyxl alone. python-calamine tolerates more malformed files,
+but openpyxl is already a dependency (column-letter arithmetic, fixtures) and one
+reader means one representation to keep golden. NOTE: if malformed workbooks show
+up in practice, calamine belongs behind an optional fallback, not a second
+default.
+"""
+
+from __future__ import annotations
+
+from decimal import Decimal
+from math import isfinite
+from pathlib import Path
+from typing import Any, Iterator
+
+from openpyxl import load_workbook
+from openpyxl.utils import get_column_letter
+
+from ..kernel.model import CellValue
+from .base import ExtractedPage, Extractor, ExtractionError, register
+
+__all__ = ["MAX_COLS", "MAX_ROWS", "XlsxExtractor", "EXTRACTOR"]
+
+MAX_ROWS = 2_000
+"""Hard cap on rows scanned per sheet."""
+
+MAX_COLS = 200
+"""Hard cap on columns scanned per sheet."""
+
+MAX_CELL_CHARS = 150
+"""A cell longer than this is truncated with an ellipsis."""
+
+_MEDIA_TYPES = frozenset({"xlsx"})
+_SUFFIXES = frozenset({".xlsx", ".xlsm", ".xltx", ".xltm"})
+
+
+class XlsxExtractor:
+    """openpyxl workbook reader. Deterministic: values in, markdown out."""
+
+    name = "xlsx"
+    version = "1"
+    deterministic = True
+
+    def can_handle(self, path: Path, media_type: str) -> bool:
+        return media_type in _MEDIA_TYPES or path.suffix.lower() in _SUFFIXES
+
+    def extract(self, path: Path, config: dict) -> Iterator[ExtractedPage]:
+        """Yield one page per sheet, in workbook order.
+
+        Sheet names are yielded as the workbook spells them; the registry
+        sanitizes them to the sheetref charset when it mints tokens.
+        """
+        try:
+            workbook = load_workbook(path, data_only=True, read_only=True)
+        except Exception as error:  # noqa: BLE001 - openpyxl raises broadly
+            raise ExtractionError(f"could not open {path}: {error}") from error
+        try:
+            for number, name in enumerate(workbook.sheetnames, start=1):
+                yield self._sheet(workbook[name], number, name)
+        finally:
+            workbook.close()
+
+    def _sheet(self, worksheet: Any, number: int, name: str) -> ExtractedPage:
+        reported_rows = worksheet.max_row or 0
+        reported_cols = worksheet.max_column or 0
+        if reported_rows > MAX_ROWS or reported_cols > MAX_COLS:
+            return ExtractedPage(
+                number=number,
+                kind="sheet",
+                name=name,
+                text=_placeholder(name, reported_rows, reported_cols),
+                cells=[],
+            )
+
+        grid = _read_grid(worksheet)
+        rows, cols = _bounds(grid)
+        if not rows or not cols:
+            return ExtractedPage(
+                number=number,
+                kind="sheet",
+                name=name,
+                text=_title(name, rows, cols, reported_rows, reported_cols),
+                cells=[],
+            )
+
+        table, cells = _render(grid, rows, cols)
+        title = _title(name, rows, cols, reported_rows, reported_cols)
+        return ExtractedPage(
+            number=number,
+            kind="sheet",
+            name=name,
+            text=f"{title}\n\n{table}",
+            cells=cells,
+        )
+
+
+def _read_grid(worksheet: Any) -> list[list[Any]]:
+    """The sheet's values, capped. Read once: read-only sheets stream."""
+    return [
+        list(row)
+        for row in worksheet.iter_rows(
+            min_row=1, max_row=MAX_ROWS, max_col=MAX_COLS, values_only=True
+        )
+    ]
+
+
+def _has_data(value: Any) -> bool:
+    """True if a cell holds data rather than formatting or emptiness."""
+    return value is not None and bool(str(value).strip())
+
+
+def _bounds(grid: list[list[Any]]) -> tuple[int, int]:
+    """The (rows, cols) actually holding data — trailing empties trimmed off."""
+    rows = 0
+    cols = 0
+    for row_index, row in enumerate(grid, start=1):
+        for col_index, value in enumerate(row, start=1):
+            if _has_data(value):
+                rows = row_index
+                cols = max(cols, col_index)
+    return rows, cols
+
+
+def _format_value(value: Any) -> str:
+    """A cell as text: integers unadorned, floats at full precision, dates ISO-8601.
+
+    Rounding here would be a correctness bug, not a display choice. The page text
+    is the snapshot and the snapshot is the receipt: a cap rate stored as
+    `0.0575` and written down as `0.058` cannot be value-traced against the claim
+    "5.75%" that was read off it, and the cell anchor's snippet — the thing a
+    reader is shown as proof — would disagree with the workbook.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        # NOTE: before the int branch — bool is an int and `str(int(True))` is "1".
+        return str(value)
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        return _float_text(value)
+    if hasattr(value, "isoformat"):
+        return value.isoformat()
+    return str(value)
+
+
+def _float_text(value: float) -> str:
+    """A float as the shortest text that reads back as the same float.
+
+    `repr` gives that shortest round-trip form, which is what keeps `0.0575` from
+    becoming `0.057499999999999996`. Routing it through `Decimal` only drops the
+    exponent, so a small rate reads as `0.000001` rather than `1e-06` — the form
+    it has in the workbook, and the form a claim would quote.
+    """
+    if not isfinite(value):
+        return repr(value)
+    if value.is_integer():
+        return str(int(value))
+    return format(Decimal(repr(value)), "f")
+
+
+def _cell_text(value: Any) -> str:
+    """A formatted cell, truncated, with markdown's `|` neutralized."""
+    text = _format_value(value).replace("\n", " ").replace("|", "\\|")
+    if len(text) > MAX_CELL_CHARS:
+        text = text[: MAX_CELL_CHARS - 3] + "..."
+    return text
+
+
+def _render(
+    grid: list[list[Any]], rows: int, cols: int
+) -> tuple[str, list[CellValue]]:
+    """The markdown table and the cell values it carries.
+
+    Every non-empty cell gets a `[B10]` prefix in the table and a `CellValue`
+    whose `value` is the cell text *as it appears in the table* — the receipt has
+    to be verbatim from the snapshot, truncation and all.
+    """
+    header = ["Row", *(get_column_letter(col) for col in range(1, cols + 1))]
+    lines = [
+        "| " + " | ".join(header) + " |",
+        "|" + "|".join(["---"] * len(header)) + "|",
+    ]
+    cells: list[CellValue] = []
+    for row_index in range(1, rows + 1):
+        row = grid[row_index - 1] if row_index <= len(grid) else []
+        rendered = [str(row_index)]
+        for col_index in range(1, cols + 1):
+            value = row[col_index - 1] if col_index <= len(row) else None
+            text = _cell_text(value)
+            if text:
+                ref = f"{get_column_letter(col_index)}{row_index}"
+                cells.append(CellValue(ref=ref, value=text))
+                text = f"[{ref}] {text}"
+            rendered.append(text)
+        lines.append("| " + " | ".join(rendered) + " |")
+    return "\n".join(lines), cells
+
+
+def _title(name: str, rows: int, cols: int, reported_rows: int, reported_cols: int) -> str:
+    """The page's heading, which says when the view is partial."""
+    note = ""
+    if reported_rows > rows or reported_cols > cols:
+        note = f" (showing {rows}x{cols} of {reported_rows}x{reported_cols})"
+    return f"## Sheet: {name}{note} - Values View with cell references"
+
+
+def _placeholder(name: str, rows: int, cols: int) -> str:
+    """The page a sheet gets when its reported dimensions are absurd."""
+    return (
+        f"## Sheet: {name}\n\n"
+        f"**Sheet could not be processed.**\n\n"
+        f"Reported dimensions ({rows:,} rows x {cols:,} columns) exceed processing "
+        f"limits ({MAX_ROWS:,} x {MAX_COLS:,}). This typically occurs when Excel "
+        f"applies formatting to empty cells, inflating the apparent sheet size."
+    )
+
+
+EXTRACTOR: Extractor = XlsxExtractor()
+register(EXTRACTOR)

backdraft/gate/__init__.py

@@ -0,0 +1,39 @@
+"""The gate: the only way source text reaches a writer's context.
+
+`read` lists, tables-of-contents and pages; `search` queries the FTS index. Both
+mint what they emit into the session ledger, which is what makes the design's
+central claim true — the set of citable tokens is exactly the set the gate
+emitted, and bind can tell a cited-what-you-saw token from a token the writer
+never had.
+
+`gate.cli` is mounted by the top-level CLI (SPEC Addendum B) and is imported
+separately, so this package stays importable without typer.
+
+Export style (shared by every package above the kernel): **names**, never
+submodules, and only what a consumer outside the package imports.
+"""
+
+from __future__ import annotations
+
+from .reader import (
+    GateError,
+    Selection,
+    read,
+    render_documents,
+    render_page_read,
+    render_toc,
+    select_pages,
+)
+from .searcher import render_search, search
+
+__all__ = [
+    "GateError",
+    "Selection",
+    "read",
+    "render_documents",
+    "render_toc",
+    "render_page_read",
+    "select_pages",
+    "search",
+    "render_search",
+]

backdraft/gate/cli.py

@@ -0,0 +1,131 @@
+"""The gate's CLI surface: `read`, `search`, `session`.
+
+Mounted by the top-level `cli.py` (SPEC Addendum B), which owns the typer app.
+This module holds no logic of its own — it parses flags, calls `reader`/
+`searcher`, prints, and lets `cli_context.guard` map a `GateError` to exit code 1
+(usage/env error).
+
+Discovery and session resolution come from `backdraft.cli_context`, imported at
+module level: that module deliberately knows nothing about the sub-apps, so the
+mount is not a cycle and there is no lazy lookup to keep in sync.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Annotated
+
+import typer
+
+from ..cli_context import SESSION_ENV, opened_registry, resolve_session
+from .reader import cells as mint_cells
+from .reader import read as read_pages
+from .searcher import search as run_search
+
+__all__ = ["app", "session_app"]
+
+app = typer.Typer(help="Read and search ingested documents through the gate.")
+session_app = typer.Typer(help="Inspect or start a ledger session.")
+app.add_typer(session_app, name="session")
+
+
+def _session_source(explicit: str | None) -> str:
+    """Which rule supplied the session id — shown by `session show`."""
+    if explicit:
+        return "--session"
+    if os.environ.get(SESSION_ENV):
+        return SESSION_ENV
+    return "default"
+
+
+# ---------------------------------------------------------------------------
+# commands
+# ---------------------------------------------------------------------------
+
+_SessionOption = Annotated[
+    str | None,
+    typer.Option("--session", "-s", help="Ledger session to mint into."),
+]
+
+
+@app.command()
+def read(
+    slug: Annotated[str | None, typer.Argument(help="Document slug.")] = None,
+    selector: Annotated[
+        str | None, typer.Argument(help="Page (`p3`), range (`p3-5`), or sheet name.")
+    ] = None,
+    session: _SessionOption = None,
+    offset: Annotated[
+        int, typer.Option("--offset", help="Units to skip (chars for pages, rows for sheets).")
+    ] = 0,
+    limit: Annotated[
+        int | None, typer.Option("--limit", help="Units to show. Default: all of them.")
+    ] = None,
+) -> None:
+    """List documents, show a document's contents, or read pages from one."""
+    # The list and the table of contents emit no tokens, so they mint nothing.
+    minting = slug is not None and selector is not None
+    session_id = resolve_session(session) if minting else None
+    with opened_registry() as registry:
+        typer.echo(
+            read_pages(
+                registry, slug, selector, session=session_id, offset=offset, limit=limit
+            )
+        )
+
+
+@app.command()
+def cell(
+    slug: Annotated[str, typer.Argument(help="Document slug.")],
+    refs: Annotated[
+        list[str], typer.Argument(help="Cell locators, like rent-roll!B10. Repeatable.")
+    ],
+    session: _SessionOption = None,
+) -> None:
+    """Mint tokens for specific cells: the token, then the verbatim value.
+
+    The direct path to citing a cell you can see in a windowed sheet read —
+    no need to search for the cell's own value.
+    """
+    session_id = resolve_session(session)
+    with opened_registry() as registry:
+        typer.echo(mint_cells(registry, slug, refs, session=session_id))
+
+
+@app.command()
+def search(
+    query: Annotated[str, typer.Argument(help="FTS5 query.")],
+    in_: Annotated[
+        str | None, typer.Option("--in", help="Restrict to one document slug.")
+    ] = None,
+    session: _SessionOption = None,
+    limit: Annotated[int, typer.Option("--limit", help="Maximum results.")] = 20,
+) -> None:
+    """Search every anchor's snippet. Results are citable without a page read."""
+    session_id = resolve_session(session)
+    with opened_registry() as registry:
+        typer.echo(run_search(registry, query, slug=in_, limit=limit, session=session_id))
+
+
+@session_app.command("start")
+def session_start(
+    id_: Annotated[
+        str | None, typer.Option("--id", help="Use this id instead of a generated one.")
+    ] = None,
+    label: Annotated[str | None, typer.Option("--label", help="Human note.")] = None,
+) -> None:
+    """Start a ledger session and print the id to export."""
+    with opened_registry() as registry:
+        session_id = registry.ensure_session(id_, label)
+        typer.echo(f"session {session_id}  started")
+        typer.echo(f"[Use it: export {SESSION_ENV}={session_id}]")
+
+
+@session_app.command("show")
+def session_show(session: _SessionOption = None) -> None:
+    """Show which session reads and searches would mint into."""
+    resolved = resolve_session(session)
+    with opened_registry() as registry:
+        typer.echo(
+            f"session {registry.ensure_session(resolved)}  (from {_session_source(session)})"
+        )