backdraft 0.2.0__py3-none-any.whl

backdraft/cli.py

@@ -0,0 +1,341 @@
+"""The CLI — the system's first substrate (SPEC.md § Addendum B).
+
+This module owns the typer app, the `init` / `ingest` / `ls` / `export`
+commands, and the mounts. The context every command shares — registry discovery,
+session resolution, the exit codes and the error guard — lives in
+`cli_context.py`, which the sub-apps import directly; the names are re-exported
+here because `cli.find_root`, `cli.open_registry` and `cli.resolve_session` are
+what the spec and the tests call the CLI's surface.
+
+The gate, bind and render workstreams each ship an `app = typer.Typer()` that is
+mounted here; each mount is guarded, so a partial checkout still runs the
+commands it does have.
+
+Exit codes: 0 clean, 1 usage or environment error, 2 `bind` completing with
+non-resolved citations (so a hook can gate on it).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Iterable
+
+import typer
+
+from .cli_context import (
+    DEFAULT_SESSION,
+    EXIT_UNRESOLVED,
+    EXIT_USAGE,
+    HOME_ENV,
+    SESSION_ENV,
+    UsageError,
+    find_root,
+    guard,
+    open_registry,
+    opened_registry,
+    resolve_session,
+)
+from .extract import vlm_ready
+from .registry import DIRECTORY, Registry
+
+__all__ = [
+    "app",
+    "main",
+    # re-exported from cli_context: this module is the CLI's front door
+    "DEFAULT_SESSION",
+    "EXIT_UNRESOLVED",
+    "EXIT_USAGE",
+    "HOME_ENV",
+    "SESSION_ENV",
+    "find_root",
+    "open_registry",
+    "resolve_session",
+]
+
+app = typer.Typer(
+    name="backdraft",
+    help="Drop-in provenance for factual claims.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+_ENV_TEMPLATE = """\
+# Backdraft reads credentials only from here, BACKDRAFT_* variables, or --config.
+# Ambient provider keys (OPENAI_API_KEY, ...) are never read.
+BACKDRAFT_VLM_API_KEY=
+BACKDRAFT_ENTAIL_API_KEY=
+"""
+
+# ---- commands ---------------------------------------------------------------
+
+
+@app.command()
+def init(
+    directory: Annotated[
+        Path | None,
+        typer.Argument(help="Project root to initialize. Defaults to the current directory."),
+    ] = None,
+) -> None:
+    """Create `.backdraft/` here and print the registry's status."""
+    root = (directory or Path.cwd()).resolve()
+    with guard():
+        registry = Registry.open(root)
+        try:
+            documents = registry.documents()
+        finally:
+            registry.close()
+    env_path = root / DIRECTORY / "env"
+    if not env_path.exists():
+        env_path.write_text(_ENV_TEMPLATE, encoding="utf-8")
+    typer.echo(f"registry: {root / DIRECTORY}")
+    typer.echo(f"documents: {len(documents)}")
+    typer.echo(f"settings:  {env_path}  (credentials go here, deliberately)")
+    typer.echo(
+        "next: ingest sources, then have an agent write against them — "
+        "the backdraft skill (skills/backdraft) is the writing contract."
+    )
+
+
+@app.command()
+def ingest(
+    files: Annotated[list[Path], typer.Argument(help="Files to ingest.")],
+    extractor: Annotated[str, typer.Option("--extractor", help="Extractor name, or `auto`.")] = (
+        "auto"
+    ),
+    slug: Annotated[
+        str | None, typer.Option("--slug", help="Slug for a new document. One file only.")
+    ] = None,
+    config: Annotated[
+        list[str] | None,
+        typer.Option("--config", help="Extractor config as `key=value`. Repeatable."),
+    ] = None,
+) -> None:
+    """Snapshot files into the registry, minting their anchors."""
+    nudge_vlm = False
+    with guard():
+        if slug is not None and len(files) > 1:
+            raise UsageError("--slug names one document; pass one file")
+        settings = _parse_config(config or [])
+        with opened_registry() as registry:
+            for path in files:
+                document = registry.ingest(
+                    path, extractor=extractor, slug=slug, config=settings
+                )
+                pages = registry.pages(document.slug)
+                typer.echo(
+                    f"{document.slug}  {document.filename}  "
+                    f"{document.media_type}  {len(pages)} pages"
+                )
+                nudge_vlm = nudge_vlm or (
+                    extractor == "auto"
+                    and document.media_type == "pdf"
+                    and not vlm_ready(settings)
+                )
+    if nudge_vlm:
+        # One line, once per invocation: `auto` fell back to the text layer,
+        # and the note names the condition that failed.
+        typer.echo(f"note: extracted with pdf-text (the embedded text layer). {_vlm_gap()}")
+
+
+@app.command("snapshot-pages")
+def snapshot_pages(
+    slug: Annotated[str, typer.Argument(help="An ingested PDF's slug.")],
+    file: Annotated[
+        Path | None,
+        typer.Option("--file", help="The source PDF, when it moved since ingest."),
+    ] = None,
+    dpi: Annotated[int, typer.Option("--dpi", help="Render resolution.")] = 200,
+) -> None:
+    """Backfill page snapshots for an already-ingested PDF. Local, no model calls.
+
+    Ingesting through the VLM extractor stores each page's image automatically;
+    this command adds them to registries created before that, or to text-layer
+    ingests, so `bind` can embed the cited pages into the artifact. Requires the
+    `[vlm]` extra for PDF rendering.
+    """
+    with guard():
+        try:
+            import io
+
+            from PIL import Image  # noqa: F401  (pdf2image needs it anyway)
+            from pdf2image import convert_from_path
+        except ImportError as error:
+            raise UsageError(
+                "snapshot-pages needs the [vlm] extra for PDF rendering: "
+                f"pip install 'backdraft[vlm]' ({error})"
+            ) from error
+        from .extract.vlm_settings import MAX_IMAGE_HEIGHT, SNAPSHOT_QUALITY
+
+        with opened_registry() as registry:
+            document = registry.document(slug)
+            if document is None:
+                raise UsageError(f"no document with slug {slug!r}")
+            if document.media_type != "pdf":
+                raise UsageError(f"{slug} is {document.media_type}, not a PDF")
+            source = file or Path(document.path)
+            if not source.is_file():
+                raise UsageError(
+                    f"source file not found at {source}; pass --file to point at it"
+                )
+            extraction_id = registry.current_extraction_id(slug)
+            if extraction_id is None:
+                raise UsageError(f"{slug} has no current extraction")
+            pages = registry.pages(slug)
+            for page in pages:
+                images = convert_from_path(
+                    str(source), dpi=dpi, fmt="png",
+                    first_page=page.number, last_page=page.number,
+                )
+                image = images[0]
+                if image.height > MAX_IMAGE_HEIGHT:
+                    scale = MAX_IMAGE_HEIGHT / image.height
+                    image = image.resize(
+                        (max(1, round(image.width * scale)), MAX_IMAGE_HEIGHT)
+                    )
+                buffer = io.BytesIO()
+                image.save(buffer, format="WEBP", quality=SNAPSHOT_QUALITY)
+                registry.save_page_image(
+                    extraction_id, page.number,
+                    data=buffer.getvalue(), format="webp",
+                    width=image.width, height=image.height,
+                )
+                typer.echo(f"{slug}  p{page.number}  {image.width}x{image.height}")
+            typer.echo(f"stored {len(pages)} page snapshot(s)")
+
+
+@app.command()
+def clean(
+    directory: Annotated[
+        Path | None,
+        typer.Argument(help="Directory to tidy. Defaults to the current directory."),
+    ] = None,
+) -> None:
+    """Tidy a working directory: move stray records into `.backdraft/records/`
+    and delete leftover `.bound.md` projections.
+
+    Everything removed or moved is regenerable with `backdraft bind`; artifacts
+    (`.backdraft.html`) and authored documents are never touched.
+    """
+    from .kernel.artifact import BOUND_SUFFIX, SIDECAR_SUFFIX, record_path
+
+    with guard():
+        base = (directory or Path.cwd()).resolve()
+        root = find_root(base)
+        touched = 0
+        for stray in sorted(base.glob(f"*{BOUND_SUFFIX}")):
+            stray.unlink()
+            typer.echo(f"removed {stray.name}")
+            touched += 1
+        for stray in sorted(base.glob(f"*{SIDECAR_SUFFIX}")):
+            doc_stem = stray.name.removesuffix(SIDECAR_SUFFIX)
+            target = record_path(root, base / f"{doc_stem}.md")
+            target.parent.mkdir(parents=True, exist_ok=True)
+            stray.replace(target)
+            typer.echo(f"moved {stray.name} -> {target.relative_to(root)}")
+            touched += 1
+        if not touched:
+            typer.echo("nothing to tidy")
+
+
+@app.command("ls")
+def list_documents() -> None:
+    """List the ingested documents: slug, filename, media type, page count."""
+    with opened_registry() as registry:
+        documents = registry.documents()
+        if not documents:
+            typer.echo("no documents ingested")
+            return
+        for document in documents:
+            pages = registry.pages(document.slug)
+            typer.echo(
+                f"{document.slug}\t{document.filename}\t{document.media_type}\t"
+                f"{len(pages)} pages"
+            )
+
+
+@app.command()
+def export(
+    out: Annotated[
+        Path | None, typer.Option("--out", "-o", help="Write here instead of stdout.")
+    ] = None,
+) -> None:
+    """Export the whole registry as JSON, every generation included."""
+    with opened_registry() as registry:
+        payload = json.dumps(registry.export_json(), indent=2, ensure_ascii=False)
+    if out is None:
+        typer.echo(payload)
+    else:
+        out.write_text(payload + "\n", encoding="utf-8")
+        typer.echo(f"wrote {out}")
+
+
+def _vlm_gap() -> str:
+    """Which condition keeps `auto` off the vision model — key, extra, or both."""
+    from .credentials import setting
+    from .extract.base import ExtractionError, get
+
+    has_key = bool(setting("BACKDRAFT_VLM_API_KEY"))
+    try:
+        get("vlm")
+        has_extra = True
+    except ExtractionError:
+        has_extra = False
+    if not has_key and not has_extra:
+        return (
+            "Glossy or scanned PDFs extract better through a vision model: "
+            "install backdraft[vlm] and set BACKDRAFT_VLM_API_KEY in .backdraft/env."
+        )
+    if not has_key:
+        return (
+            "The [vlm] extra is installed but BACKDRAFT_VLM_API_KEY is not set — "
+            "add it to .backdraft/env to use the vision model."
+        )
+    return (
+        "BACKDRAFT_VLM_API_KEY is set but the [vlm] extra is not installed — "
+        "install backdraft[vlm] to use the vision model."
+    )
+
+
+def _parse_config(pairs: Iterable[str]) -> dict:
+    """`key=value` strings into a config dict. Values stay strings."""
+    settings: dict[str, str] = {}
+    for pair in pairs:
+        key, separator, value = pair.partition("=")
+        if not separator or not key:
+            raise UsageError(f"--config expects key=value, got {pair!r}")
+        settings[key] = value
+    return settings
+
+
+# ---- sub-app mounts ---------------------------------------------------------
+
+
+def _mount(module: str) -> bool:
+    """Merge a workstream's sub-app into this one. False if it isn't built yet.
+
+    The spec's commands are flat — `backdraft read`, not `backdraft gate read` —
+    so a sub-app's commands are adopted rather than nested. A missing module is a
+    partial checkout, not an error.
+    """
+    try:
+        sub = __import__(module, fromlist=["app"]).app
+    except ImportError:
+        return False
+    app.registered_commands.extend(sub.registered_commands)
+    app.registered_groups.extend(sub.registered_groups)
+    return True
+
+
+for _module in ("backdraft.gate.cli", "backdraft.bind.cli", "backdraft.render.cli"):
+    _mount(_module)
+
+
+def main() -> None:
+    """Console-script entry point."""
+    app()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

backdraft/cli_context.py

@@ -0,0 +1,152 @@
+"""What every command needs before it can do anything: a registry and a session.
+
+Discovery, session resolution, the exit codes and the one error guard live here
+rather than in `cli.py` so that the sub-apps can import them the ordinary way.
+The top level imports `gate.cli`, `bind.cli` and `render.cli` in order to mount
+them (SPEC Addendum B), so anything they read off `cli.py` has to be fetched
+lazily inside each command — which is how three copies of the same lookup grew.
+This module breaks the cycle instead: it imports typer, the kernel and the
+registry, and **never a sub-app**, so it is importable from anywhere.
+
+`cli.py` re-exports these names, because `cli.find_root` / `cli.open_registry` /
+`cli.resolve_session` are what the spec, the tests and the docs already call the
+CLI's own surface. This module is the definition; `cli.py` is the front door.
+
+Exit codes (SPEC § CLI): 0 clean, 1 usage or environment error, 2 bind completed
+with non-resolved citations.
+"""
+
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from pathlib import Path
+from typing import TYPE_CHECKING, NoReturn
+
+import typer
+
+from .kernel.errors import BackdraftError
+from .registry import DIRECTORY, Registry
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+__all__ = [
+    "DEFAULT_SESSION",
+    "EXIT_USAGE",
+    "EXIT_UNRESOLVED",
+    "HOME_ENV",
+    "SESSION_ENV",
+    "UsageError",
+    "fail",
+    "find_root",
+    "guard",
+    "open_registry",
+    "opened_registry",
+    "resolve_session",
+]
+
+HOME_ENV = "BACKDRAFT_HOME"
+SESSION_ENV = "BACKDRAFT_SESSION"
+
+DEFAULT_SESSION = "default"
+"""The auto-created session. Stable across invocations, so reads accumulate."""
+
+EXIT_USAGE = 1
+"""Usage or environment error."""
+
+EXIT_UNRESOLVED = 2
+"""`bind` completed and something did not resolve. Owned by `bind`; hooks gate on it."""
+
+
+class UsageError(BackdraftError):
+    """The command cannot run as asked: no registry, no such file, bad flag.
+
+    A domain error like any other, so it travels out of a helper and is turned
+    into a message and an exit code in exactly one place (`guard`).
+    """
+
+
+# ---- discovery --------------------------------------------------------------
+
+
+def find_root(start: Path | None = None) -> Path | None:
+    """The nearest directory containing `.backdraft/`, walking up from `start`.
+
+    `BACKDRAFT_HOME` overrides the walk. It may name either the project root or
+    the `.backdraft` directory itself — NOTE: the spec does not say which, and
+    guessing wrong is a confusing failure, so both are accepted.
+    """
+    override = os.environ.get(HOME_ENV)
+    if override:
+        home = Path(override).expanduser()
+        return home.parent if home.name == DIRECTORY else home
+    current = (start or Path.cwd()).resolve()
+    for candidate in (current, *current.parents):
+        if (candidate / DIRECTORY).is_dir():
+            return candidate
+    return None
+
+
+def open_registry(start: Path | None = None) -> Registry:
+    """Open the discovered registry, or raise `UsageError` for `guard` to report."""
+    root = find_root(start)
+    if root is None:
+        raise UsageError(
+            f"no {DIRECTORY}/ found in this directory or any parent; run `backdraft init`"
+        )
+    return Registry.open(root)
+
+
+def resolve_session(session: str | None = None, registry: Registry | None = None) -> str:
+    """The session this invocation belongs to.
+
+    Precedence: `--session` flag, then `BACKDRAFT_SESSION`, then the default
+    session. Pure name resolution — pass `registry` to also create the session
+    row. The gate's reader/searcher ensure the session themselves, so their CLI
+    passes no registry. Shared with the gate and bind sub-apps so every command
+    agrees on which ledger it is writing to.
+    """
+    chosen = session or os.environ.get(SESSION_ENV) or DEFAULT_SESSION
+    if registry is not None:
+        registry.ensure_session(chosen)
+    return chosen
+
+
+# ---- the error path ---------------------------------------------------------
+
+
+def fail(message: str, code: int = EXIT_USAGE) -> NoReturn:
+    """One line on stderr, then exit. The only place a command prints an error."""
+    typer.echo(f"backdraft: {message}", err=True)
+    raise typer.Exit(code)
+
+
+@contextmanager
+def guard(code: int = EXIT_USAGE) -> Iterator[None]:
+    """Turn any `BackdraftError` raised inside into a message and an exit code.
+
+    Libraries raise; the CLI maps — once, here. Every subclass is covered, so a
+    `GateError`, an `ExtractionError`, a `RegistryError` and a `UsageError` all
+    reach the user as one line rather than a traceback, and a command that wants
+    a different exit code passes one instead of writing its own handler.
+    """
+    try:
+        yield
+    except BackdraftError as error:
+        fail(str(error), code)
+
+
+@contextmanager
+def opened_registry(start: Path | None = None) -> Iterator[Registry]:
+    """The discovered registry, guarded and always closed.
+
+    The shape almost every command wants: discovery failures and domain errors
+    raised in the body both become exit 1, and the connection closes either way.
+    """
+    with guard():
+        registry = open_registry(start)
+        try:
+            yield registry
+        finally:
+            registry.close()

backdraft/credentials.py

@@ -0,0 +1,106 @@
+"""Backdraft-scoped settings: the only way a credential reaches this tool.
+
+The rule, decided after a real incident: **backdraft never reads a credential it
+did not explicitly ask you to write.** Ambient provider variables — an
+`OPENAI_API_KEY` exported for some other tool, an `ANTHROPIC_API_KEY` in a
+dotfile — are never consulted. Presence of a generic key in the environment is
+not consent to spend it or to send documents to its provider.
+
+A setting is looked up in three places, most explicit first:
+
+1. the `--config key=value` dict a command was invoked with;
+2. a `BACKDRAFT_<NAME>` environment variable — still env, but exporting a
+   backdraft-prefixed variable is unambiguous intent *for this tool*;
+3. a `.backdraft/env` file — plain `KEY=VALUE` lines, written deliberately,
+   living inside the state directory that is already gitignored.
+
+Names in use: `BACKDRAFT_VLM_API_KEY`, `BACKDRAFT_VLM_MODEL`,
+`BACKDRAFT_VLM_BASE_URL`, `BACKDRAFT_ENTAIL_API_KEY`, `BACKDRAFT_ENTAIL_MODEL`.
+The same names are used verbatim in the env file.
+
+Stdlib only, no imports from the package: this module sits beside the kernel so
+that `extract` and `bind` can both use it without a sideways import.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+__all__ = ["setting", "env_file", "ENV_FILE_NAME"]
+
+ENV_FILE_NAME = "env"
+_STATE_DIR = ".backdraft"
+_PREFIX = "BACKDRAFT_"
+
+
+def setting(
+    name: str,
+    config: dict | None = None,
+    *,
+    config_key: str | None = None,
+    start: Path | None = None,
+) -> str | None:
+    """The value of the backdraft-scoped setting `name`, or None.
+
+    `name` is the full variable name (`BACKDRAFT_VLM_API_KEY`). `config_key`
+    names the `--config` dict key that overrides it (`api_key`); omitted means
+    the setting has no per-invocation form. Precedence: config → environment →
+    `.backdraft/env` discovered from `start` (default: cwd, honoring
+    `BACKDRAFT_HOME`).
+    """
+    if not name.startswith(_PREFIX):  # pragma: no cover - caller bug, fail loud
+        raise ValueError(f"not a backdraft-scoped name: {name!r}")
+    if config_key is not None and config and config.get(config_key):
+        return str(config[config_key])
+    if value := os.environ.get(name):
+        return value
+    return _read_env_file(start).get(name)
+
+
+def env_file(start: Path | None = None) -> Path | None:
+    """The `.backdraft/env` file that governs `start` (default cwd), if any.
+
+    `BACKDRAFT_HOME` overrides discovery, accepting either the project root or
+    the `.backdraft` directory itself — the same rule the CLI uses for the
+    registry.
+    """
+    override = os.environ.get("BACKDRAFT_HOME")
+    if override:
+        home = Path(override).expanduser()
+        root = home.parent if home.name == _STATE_DIR else home
+        candidate = root / _STATE_DIR / ENV_FILE_NAME
+        return candidate if candidate.is_file() else None
+    current = (start or Path.cwd()).resolve()
+    for parent in (current, *current.parents):
+        candidate = parent / _STATE_DIR / ENV_FILE_NAME
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def _read_env_file(start: Path | None) -> dict[str, str]:
+    """Parse the governing env file: `KEY=VALUE` lines, `#` comments, optional
+    single or double quotes around the value. Unreadable or absent → empty."""
+    path = env_file(start)
+    if path is None:
+        return {}
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return {}
+    values: dict[str, str] = {}
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        key, separator, value = line.partition("=")
+        if not separator:
+            continue
+        key = key.strip()
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        if key.startswith(_PREFIX):
+            values[key] = value
+    return values

backdraft/extract/__init__.py

@@ -0,0 +1,37 @@
+"""Extractors: files in, ordered pages out.
+
+The protocol and the registry live in `base`; the built-ins (`text`, `pdf-text`,
+`xlsx`) and the `[vlm]` extra are imported on demand by `base.get`, so importing
+this package costs nothing but the protocol.
+
+Export style (shared by every package above the kernel): **names**, never
+submodules, and only what a consumer outside the package imports.
+"""
+
+from __future__ import annotations
+
+from .base import (
+    AUTO_ORDER,
+    EXTRACTORS,
+    ExtractedPage,
+    ExtractionError,
+    Extractor,
+    get,
+    names,
+    register,
+    select,
+    vlm_ready,
+)
+
+__all__ = [
+    "AUTO_ORDER",
+    "EXTRACTORS",
+    "ExtractedPage",
+    "Extractor",
+    "ExtractionError",
+    "get",
+    "names",
+    "register",
+    "select",
+    "vlm_ready",
+]