backdraft 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. backdraft-0.2.0/.gitignore +24 -0
  2. backdraft-0.2.0/.python-version +1 -0
  3. backdraft-0.2.0/DESIGN.md +146 -0
  4. backdraft-0.2.0/LICENSE +21 -0
  5. backdraft-0.2.0/PKG-INFO +253 -0
  6. backdraft-0.2.0/README.md +236 -0
  7. backdraft-0.2.0/ROADMAP.md +79 -0
  8. backdraft-0.2.0/SPEC.md +365 -0
  9. backdraft-0.2.0/assets/backdraft-mark.svg +11 -0
  10. backdraft-0.2.0/demo/generate_sources.py +331 -0
  11. backdraft-0.2.0/demo/memo.backdraft.html +1808 -0
  12. backdraft-0.2.0/demo/memo.md +67 -0
  13. backdraft-0.2.0/demo/sources/t12-summary.pdf +118 -0
  14. backdraft-0.2.0/demo/sources/underwriting-model.xlsx +0 -0
  15. backdraft-0.2.0/demo/walkthrough.md +527 -0
  16. backdraft-0.2.0/pyproject.toml +41 -0
  17. backdraft-0.2.0/research/20260727-excel-region-detection.md +53 -0
  18. backdraft-0.2.0/research/20260727-research-findings.md +182 -0
  19. backdraft-0.2.0/site/demo.html +1808 -0
  20. backdraft-0.2.0/site/docs.html +312 -0
  21. backdraft-0.2.0/site/index.html +410 -0
  22. backdraft-0.2.0/site/llms.txt +68 -0
  23. backdraft-0.2.0/skills/backdraft/SKILL.md +147 -0
  24. backdraft-0.2.0/skills/backdraft-artifact/SKILL.md +92 -0
  25. backdraft-0.2.0/skills/backdraft-backfill/SKILL.md +137 -0
  26. backdraft-0.2.0/spec/artifact.md +279 -0
  27. backdraft-0.2.0/spec/chunking.md +63 -0
  28. backdraft-0.2.0/spec/tokens.md +95 -0
  29. backdraft-0.2.0/src/backdraft/__init__.py +13 -0
  30. backdraft-0.2.0/src/backdraft/bind/__init__.py +37 -0
  31. backdraft-0.2.0/src/backdraft/bind/binder.py +653 -0
  32. backdraft-0.2.0/src/backdraft/bind/cli.py +133 -0
  33. backdraft-0.2.0/src/backdraft/bind/evidence.py +232 -0
  34. backdraft-0.2.0/src/backdraft/bind/verify/__init__.py +30 -0
  35. backdraft-0.2.0/src/backdraft/bind/verify/base.py +70 -0
  36. backdraft-0.2.0/src/backdraft/bind/verify/entail.py +156 -0
  37. backdraft-0.2.0/src/backdraft/bind/verify/overlap.py +114 -0
  38. backdraft-0.2.0/src/backdraft/bind/verify/recompute.py +45 -0
  39. backdraft-0.2.0/src/backdraft/bind/verify/value_trace.py +352 -0
  40. backdraft-0.2.0/src/backdraft/cli.py +341 -0
  41. backdraft-0.2.0/src/backdraft/cli_context.py +152 -0
  42. backdraft-0.2.0/src/backdraft/credentials.py +106 -0
  43. backdraft-0.2.0/src/backdraft/extract/__init__.py +37 -0
  44. backdraft-0.2.0/src/backdraft/extract/base.py +184 -0
  45. backdraft-0.2.0/src/backdraft/extract/pdf_text.py +124 -0
  46. backdraft-0.2.0/src/backdraft/extract/text.py +44 -0
  47. backdraft-0.2.0/src/backdraft/extract/vlm.py +175 -0
  48. backdraft-0.2.0/src/backdraft/extract/vlm_settings.py +197 -0
  49. backdraft-0.2.0/src/backdraft/extract/xlsx.py +244 -0
  50. backdraft-0.2.0/src/backdraft/gate/__init__.py +39 -0
  51. backdraft-0.2.0/src/backdraft/gate/cli.py +131 -0
  52. backdraft-0.2.0/src/backdraft/gate/reader.py +592 -0
  53. backdraft-0.2.0/src/backdraft/gate/searcher.py +123 -0
  54. backdraft-0.2.0/src/backdraft/kernel/__init__.py +17 -0
  55. backdraft-0.2.0/src/backdraft/kernel/artifact.py +234 -0
  56. backdraft-0.2.0/src/backdraft/kernel/chunking.py +162 -0
  57. backdraft-0.2.0/src/backdraft/kernel/claims.py +142 -0
  58. backdraft-0.2.0/src/backdraft/kernel/errors.py +35 -0
  59. backdraft-0.2.0/src/backdraft/kernel/hashing.py +89 -0
  60. backdraft-0.2.0/src/backdraft/kernel/model.py +336 -0
  61. backdraft-0.2.0/src/backdraft/kernel/tokens.py +291 -0
  62. backdraft-0.2.0/src/backdraft/registry/__init__.py +39 -0
  63. backdraft-0.2.0/src/backdraft/registry/ledger.py +82 -0
  64. backdraft-0.2.0/src/backdraft/registry/schema.sql +90 -0
  65. backdraft-0.2.0/src/backdraft/registry/store.py +915 -0
  66. backdraft-0.2.0/src/backdraft/render/__init__.py +31 -0
  67. backdraft-0.2.0/src/backdraft/render/_text.py +69 -0
  68. backdraft-0.2.0/src/backdraft/render/cli.py +107 -0
  69. backdraft-0.2.0/src/backdraft/render/footnotes.py +172 -0
  70. backdraft-0.2.0/src/backdraft/render/html.py +1211 -0
  71. backdraft-0.2.0/src/backdraft/render/markdown.py +313 -0
  72. backdraft-0.2.0/src/backdraft/render/placement.py +105 -0
  73. backdraft-0.2.0/src/backdraft/render/sidecar.py +164 -0
  74. backdraft-0.2.0/tests/conftest.py +9 -0
  75. backdraft-0.2.0/tests/conftest_registry.py +135 -0
  76. backdraft-0.2.0/tests/conftest_render.py +231 -0
  77. backdraft-0.2.0/tests/fakes.py +175 -0
  78. backdraft-0.2.0/tests/gate/conftest.py +62 -0
  79. backdraft-0.2.0/tests/gate/fake_registry.py +311 -0
  80. backdraft-0.2.0/tests/gate/test_gate_cli.py +191 -0
  81. backdraft-0.2.0/tests/gate/test_gate_ledger.py +107 -0
  82. backdraft-0.2.0/tests/gate/test_gate_reader.py +360 -0
  83. backdraft-0.2.0/tests/gate/test_gate_searcher.py +128 -0
  84. backdraft-0.2.0/tests/gate/test_reader_cells.py +31 -0
  85. backdraft-0.2.0/tests/golden/bind/backfill.bound.md +18 -0
  86. backdraft-0.2.0/tests/golden/bind/frontwalk.bound.md +23 -0
  87. backdraft-0.2.0/tests/golden/chunking/long_tail.json +14 -0
  88. backdraft-0.2.0/tests/golden/chunking/long_tail.txt +1 -0
  89. backdraft-0.2.0/tests/golden/chunking/many_small.json +20 -0
  90. backdraft-0.2.0/tests/golden/chunking/many_small.txt +23 -0
  91. backdraft-0.2.0/tests/golden/chunking/mixed.json +20 -0
  92. backdraft-0.2.0/tests/golden/chunking/mixed.txt +7 -0
  93. backdraft-0.2.0/tests/golden/chunking/one_giant.json +26 -0
  94. backdraft-0.2.0/tests/golden/chunking/one_giant.txt +1 -0
  95. backdraft-0.2.0/tests/golden/chunking/short_page.json +8 -0
  96. backdraft-0.2.0/tests/golden/chunking/short_page.txt +3 -0
  97. backdraft-0.2.0/tests/golden/chunking/unsplittable.json +8 -0
  98. backdraft-0.2.0/tests/golden/chunking/unsplittable.txt +1 -0
  99. backdraft-0.2.0/tests/golden/render/backfill.footnotes.md +27 -0
  100. backdraft-0.2.0/tests/golden/render/backfill.sidecar.json +93 -0
  101. backdraft-0.2.0/tests/golden/render/demo.footnotes.md +104 -0
  102. backdraft-0.2.0/tests/golden/render/demo.sidecar.json +229 -0
  103. backdraft-0.2.0/tests/golden_util.py +43 -0
  104. backdraft-0.2.0/tests/test_bind_binder.py +637 -0
  105. backdraft-0.2.0/tests/test_bind_cli.py +216 -0
  106. backdraft-0.2.0/tests/test_bind_evidence.py +141 -0
  107. backdraft-0.2.0/tests/test_bind_verify.py +230 -0
  108. backdraft-0.2.0/tests/test_chunking.py +208 -0
  109. backdraft-0.2.0/tests/test_claims.py +189 -0
  110. backdraft-0.2.0/tests/test_cli.py +178 -0
  111. backdraft-0.2.0/tests/test_credentials.py +67 -0
  112. backdraft-0.2.0/tests/test_extract.py +627 -0
  113. backdraft-0.2.0/tests/test_footnotes.py +113 -0
  114. backdraft-0.2.0/tests/test_gate_integration.py +91 -0
  115. backdraft-0.2.0/tests/test_hashing.py +171 -0
  116. backdraft-0.2.0/tests/test_html.py +454 -0
  117. backdraft-0.2.0/tests/test_invariants.py +178 -0
  118. backdraft-0.2.0/tests/test_model.py +213 -0
  119. backdraft-0.2.0/tests/test_placement.py +54 -0
  120. backdraft-0.2.0/tests/test_records_location.py +51 -0
  121. backdraft-0.2.0/tests/test_registry.py +479 -0
  122. backdraft-0.2.0/tests/test_reingest.py +217 -0
  123. backdraft-0.2.0/tests/test_render_cli.py +161 -0
  124. backdraft-0.2.0/tests/test_render_markdown.py +121 -0
  125. backdraft-0.2.0/tests/test_sidecar.py +207 -0
  126. backdraft-0.2.0/tests/test_tokens.py +279 -0
  127. backdraft-0.2.0/tests/test_value_trace.py +279 -0
  128. backdraft-0.2.0/uv.lock +812 -0
@@ -0,0 +1,24 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ .Python
5
+ build/
6
+ dist/
7
+ *.egg-info/
8
+ .venv/
9
+ venv/
10
+
11
+ # Tooling
12
+ .pytest_cache/
13
+ .ruff_cache/
14
+ .mypy_cache/
15
+ uv.lock.bak
16
+
17
+ # Backdraft registry
18
+ .backdraft/
19
+
20
+ # OS
21
+ .DS_Store
22
+
23
+ # Claude Code session state and agent worktrees
24
+ .claude/
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,146 @@
1
+ # Backdraft — Design
2
+
3
+ Status: v0 design, 2026-07-27. Decisions below are made unless marked open. Research backing this is in [research/20260727-research-findings.md](research/20260727-research-findings.md).
4
+
5
+ ## The point
6
+
7
+ Drop-in provenance for factual claims: one click from any claim to the evidence behind it. Extracted from what was built inside the HighRoad app, redesigned from scratch, released standalone. The working problem statement is **verification cost** — an analyst who can't show where a number came from either re-reads the document or doesn't use the output, and both outcomes destroy the value of extraction.
8
+
9
+ Not a port. The app proved the product (citations worked great from the frontend) but its mechanism was path-dependent: random UUIDs, four citation grammars, four resolver copies, silent failure everywhere, anchors that die on reprocessing. We port the ideas, not the code.
10
+
11
+ ## Principles
12
+
13
+ 1. **Gate, don't verify.** The structure of how information reaches the model determines what it can cite. Reading happens through a tool that mints citation tokens; the set of citable things is exactly the set of things shown. Verification (below) is a separate, optional layer — not the correctness story.
14
+ 2. **Content-addressed everything.** Anchors derive their identity from document content + location, not from a run. Reprocessing the same bytes yields the same anchors; citations survive.
15
+ 3. **The receipt travels with the claim.** An anchor is not a pointer — it is a pointer plus captured evidence: verbatim snippet, content hash, location. Defensible without reopening the source.
16
+ 4. **Failures are data.** Unresolvable tokens, drifted sources, unmatched claims are first-class records in reports and artifacts — never warn-and-drop.
17
+ 5. **Coordinates live in-band.** The app's only working sub-page attribution was Excel, because `[B10]` markers sat in the text the model read. Human-readable location in the token; side-channel geometry rots (bbox was plumbed through six layers and read by nothing).
18
+ 6. **The format is the product.** Substrates (skill, CLI, SDK, viewer) are thin shells over one spec + one library. Artifacts are self-describing and outlive any substrate (subtext's `$format` + embedded legend pattern).
19
+
20
+ ## Architecture: three layers
21
+
22
+ ### 1. The spec
23
+ - **Citation token grammar** — one grammar, typed, shared parser (Python + TS).
24
+ - **Anchor schema** — pointer + receipt.
25
+ - **Artifact format** — self-describing annotated document: versioned format string, embedded legend, claims with citation lists, sources map, provenance slot (filled — `author_type: human|agent`, model, run; the field subtext specified and left empty), verification records, unresolved list.
26
+
27
+ ### 2. The library / CLI (the system)
28
+ Four verbs:
29
+ - **`ingest`** — deterministically anchor source documents (PDF/xlsx/md/…) into the registry: extract, locate, snapshot snippets, hash.
30
+ - **`read` / `search`** — the gate. Emit token-marked context for a model. Search results are themselves mintable evidence (fixing pipeline-v2's gap where search returned nothing citable and forced a page read).
31
+ - **`bind`** — postprocess a written document: resolve tokens, run enabled verification switches, backfill surface forms (doc names, pages, quotes), emit sidecar claims file + binding report.
32
+ - **`render`** — produce the clickable artifact.
33
+
34
+ ### 3. Substrates (sequenced)
35
+ 1. **Skill + CLI, together** (first release). The CLI is the system; the skill is ~a page instructing an agent to use it. Ships both the agent-native wedge and the zero-integration backfill demo.
36
+ 2. **SDK middleware** — wrap a model call: minted context in, bound citations out. The app-developer drop-in (LoanCore validation-layer shape).
37
+ 3. **Viewer** — the artifact carries its own single-file viewer; a component version for embedding comes later.
38
+
39
+ ## The registry
40
+
41
+ **SQLite**, in `.backdraft/` per project. JSON export command for portability and diffing.
42
+
43
+ Tables (shape, not final DDL):
44
+ - `documents` — path, display name, **slug**, content sha256, media type, ingest metadata.
45
+ - `anchors` — token, document, locator, verbatim snippet, snippet sha256, created-by (ingest run).
46
+ - `ledger` — session id, token, when shown. Records every token minted into context. Bind distinguishes "cited what you saw" from "cited a valid token you were never shown" — a caught hallucination class the app couldn't express.
47
+ - `claims` (written at bind) — claim span text, source doc, citation tokens, verification records.
48
+
49
+ FTS5 backs `search`. No server, no tenancy — single-analyst/single-project scope; multi-user is a substrate concern, not a registry concern.
50
+
51
+ ## Token grammar
52
+
53
+ **`bd:<slug>:<locator>:<hash>`** — compound, human-readable location, content-hash suffix.
54
+
55
+ - `slug` — short kebab doc identifier assigned at ingest, unique per registry (`t12-audit`, `rent-roll-2025`).
56
+ - `locator` — media-native:
57
+ - PDF/text: `p8` (page), `p8.c3` (chunk ordinal within page; chunking is deterministic with an explicit within-page ordinal — the column the app never had)
58
+ - Excel: `Rent!B10`, `Rent!B10:C12` (sheet + cell/range, spreadsheet-native)
59
+ - Prose files: heading-path or line-range forms (open detail)
60
+ - `hash` — first 4+ hex of sha256 of the normalized snippet. Catches transcription typos, detects drift after re-ingest, and makes the token content-addressed. Registry extends length on collision.
61
+
62
+ Examples: `[DSCR of 1.42x](bd:t12-audit:p8.c3:a7f3)` · `[NOI of $4.1M](bd:model:Rent!B10:9e2f)`
63
+
64
+ Reserved extension — **declared derivations**: a claim whose value appears in no document carries its computation over multiple tokens (`bd:calc(model:Rent!B10 / t12-audit:p4.c1)` — sketch, grammar open). This is what makes "the numbers tie" a deterministic check instead of a slogan; only possible because we own the grammar.
65
+
66
+ ## The gate (context construction)
67
+
68
+ Ported ideas from pipeline-v2's read tool, rebuilt local:
69
+ - Unified dispatch: list docs → per-doc TOC of page/sheet summaries → page/range read → batch read. Short slugs everywhere.
70
+ - Progressive disclosure with continuation, plus search hints.
71
+ - **Excel**: sheet-per-page markdown table with `Row | A | B | …` headers and `[B10] value` cell prefixes — the representation that made Excel the app's only working sub-page attribution. Hardening kept: dimension caps, inflated-sheet placeholders, trimming.
72
+ - Fixes over the app: search mints citable snippet tokens; Excel long-context gets structural navigation (detected regions/ranges) instead of char-offset slicing (open design area).
73
+
74
+ The skill's core instruction is a substitution: for source documents, use `backdraft read`/`search`, not raw Read/Grep.
75
+
76
+ ## Wire format
77
+
78
+ The writer emits inline tokens on the claim span — `[claim text](bd:…)` — and nothing else: no doc names, no footnotes, no display text. Span binding (which words the anchor supports) is captured at write time because it can never be recovered later and can always be projected away. Footnotes, endnote lists, hover cards are all render-time projections.
79
+
80
+ ## Bind
81
+
82
+ Runs after writing (in Claude Code: enforceable as a Stop/PostToolUse hook so unbound tokens can't ship).
83
+
84
+ 1. **Resolve** every token: parses → registry hit → ledger hit → snippet hash matches current source. Resolution is inherent to bind (an unresolvable token can't be backfilled) — it is not a switch.
85
+ 2. **Verify** per enabled switches (below).
86
+ 3. **Backfill** surface forms: tokens become readable citations (doc name, page/cell, quote) in the output doc.
87
+ 4. **Report**: sidecar claims file + binding report. Every failure is a line item: `unresolved`, `not-shown`, `drifted`, `unmatched` (backfill mode). Nothing drops silently.
88
+
89
+ ## Verification — switches, default off
90
+
91
+ Independent checks per citation, recorded as graded evidence, never gating generation. The record says which methods ran and what each found; switches off ⇒ record says unchecked. Grade schema is stable regardless of which switches are on.
92
+
93
+ | # | Switch | Method | Claim class |
94
+ |---|---|---|---|
95
+ | 1 | `value-trace` | Deterministic: normalized value (units/scale/format) occurs at the cited location | numbers, dates, names |
96
+ | 2 | `overlap` | Heuristic span overlap / exact substring for quotes — report-only signal | quotes, paraphrase |
97
+ | 3 | `entail` | Model judge: does snippet support claim — tiny closed question, batchable, async | qualitative |
98
+ | 4 | `recompute` | Deterministic re-execution of declared derivations | derived values |
99
+
100
+ Rationale for default-off: out of the box this is pure provenance (one click to the receipt); verification is opt-in per claim class. Rung-pass rates aggregated per run/model double as an eval substrate (the Clarion use).
101
+
102
+ ## Modes
103
+
104
+ - **Front-walk** — citations minted during generation; bind resolves against the session ledger. The high-quality integrated path (what the app did).
105
+ - **Backfill** — an existing document + ingested sources; for each claim: search → propose anchors → bind via the fuzzier strategies (overlap, value-trace, structured lookup). Unmatched claims land in an explicit unresolved list, not silent non-attribution. Zero integration required — the demo wedge.
106
+
107
+ Same anchors, same bind, same artifact out.
108
+
109
+ ## The artifact
110
+
111
+ Self-contained HTML (plus the machine-readable sidecar): **quotes + receipts + evidence embedded** — verbatim snippets, hashes, locations, verification grades, and the evidence itself: cited pages as images, cited cells in mini-grid windows. Bounded by what is cited, never the corpus; `--lean` drops images. Small and portable; defensible with sources absent. Designed in three disclosure layers (document → footnote → record) with success silent and failure loud — see the 2026-07-28 decision rows.
112
+
113
+ ## Decision log
114
+
115
+ | Date | Decision |
116
+ |---|---|
117
+ | 2026-07-27 | Rewrite, not port; ideas only |
118
+ | 2026-07-27 | Gate-over-verify: read path mints citable tokens; ledger records what was shown |
119
+ | 2026-07-27 | Receipt (snippet + hash + location) is part of the anchor primitive |
120
+ | 2026-07-27 | Registry: SQLite in `.backdraft/`, JSON export |
121
+ | 2026-07-27 | Token grammar: compound `bd:slug:loc:hash` |
122
+ | 2026-07-27 | Verification: independent switches, default off; resolution is inherent to bind, not a switch |
123
+ | 2026-07-27 | First substrate: skill + CLI together; SDK middleware and viewer follow |
124
+ | 2026-07-27 | Artifact: quotes + receipts embedded, no page images in v0 |
125
+ | 2026-07-27 | VLM extraction is the recommended primary path for real PDFs (glossy layouts, info boxes — the snapshot is the receipt, and model reading beats scrambled text-layer order). Text-layer is the keyless floor and CI path, not "the default." `ingest --extractor auto` picks VLM when an API key is configured, else text-layer with a printed nudge. Known tradeoff, accepted: re-extraction under a non-deterministic extractor drifts all citations; generations + drift reporting carry that honestly. |
126
+ | 2026-07-27 | Kernel API is module paths (`backdraft.kernel.tokens.parse`); no flat re-export surface. Package `__init__`s export exactly what outside consumers import. |
127
+ | 2026-07-27 | CLI unified on the gate's patterns: shared `cli_context` (constants, discovery, sessions, one error guard mapping `BackdraftError` → exit 1), `Annotated` params, relative imports. Test fakes renamed so the real and fake registries cannot be confused. |
128
+ | 2026-07-27 | File naming is part of the artifact format, owned by `kernel/artifact.py`: `<stem>.bound.md`, `<stem>.backdraft.json`, `<stem>.backdraft.html`, `<stem>.footnotes.md`. |
129
+ | 2026-07-27 | `bind --check entail` without the extra records `skip` verdicts carrying the reason — verifiers never gate, including at the CLI layer. |
130
+ | 2026-07-27 | **Supersedes "VLM when key present":** credentials are explicit. Ambient provider keys (`OPENAI_API_KEY`, `OPENROUTER_API_KEY`, `ANTHROPIC_API_KEY`) are never read; a credential reaches backdraft only via `--config`, a `BACKDRAFT_*` variable, or `.backdraft/env` (`init` writes a template). `vlm_ready` — the `auto` gate — is therefore equivalent to consent. Decided after a real unapproved-spend incident during the Darnell trial: presence of a generic key in the environment is not consent to spend it or send documents to its provider. |
131
+ | 2026-07-28 | **Reader doctrine (supersedes "no script"):** the artifact is designed for two readers — the writer verifying the AI's work, and a normal person the document is sent to — in three disclosure layers: the document (editorial typography, quiet marks), the footnote (click a claim → the source's own words, named and paged — no tokens, no hashes, no jargon), and the record (one deliberate step to the full machine-readable layer). **Success is silent; only failure speaks**: a fully-resolved artifact looks like a well-made document, while unresolved citations announce themselves before the reader invests belief. The load-bearing constraint is *no network, single file* — enforced by a CSP `meta` tag (`default-src 'none'`) — not *no script*: inline JS is allowed as progressive enhancement over a CSS-only baseline that survives script-stripping. Decided after field use: the v1 artifact rendered success and failure at equal volume and read as machinery, not a document. |
132
+ | 2026-07-28 | **Evidence embeds in the artifact (supersedes "no page images in v0"):** the footnote layer carries the evidence itself, not just the quote — the cited page as an image, the cited cell in a mini-grid window (neighbors, row labels, column headers). Bounded by *what is cited*, never the corpus, so a memo citing ten pages costs ~2–3 MB; `--lean` skips images. Evidence is assembled at **bind** (the step that has the registry) and travels in the sidecar; render stays registry-blind and the artifact stays reproducible from the two files a reader was handed. For the VLM path, the page image the model was shown is stored at **ingest** — for a non-deterministic extractor the input pixels are the only reproducible snapshot, so the image completes the receipt chain (claim → quote → page). Reference shapes ported from embed-ses `generate-preview`: WebP quality 85 for pages; sheet windows as compact JSON with typed cell values and column-letter headers. |
133
+ | 2026-07-28 | **Exhibits: the evidence-first inversion.** The artifact carries a second reading mode: every source document with the data points cited from it, each with its evidence context and backlinks to the claims that lean on it — "this memo rests on N data points across M documents." Ranking is deterministic in v1 (an anchor cited by three claims outranks one cited once); model-judged importance is a later switch. This is the skeptical reader's entry point and the writer's spot-check checklist. |
134
+
135
+ ## Open
136
+
137
+ Queued work lives in ROADMAP.md (exhibits, region maps, formats, `bd:calc`,
138
+ substrates); this list is questions still awaiting a decision:
139
+
140
+ - Re-bind/orphan pass on re-ingest of changed docs (chunk ordinal drift)
141
+ - Prose-file locator forms; slug assignment/collision rules (the trailing-hyphen
142
+ truncation from the Darnell trial lives here)
143
+ - Distribution: PyPI (name `backdraft` verified free 2026-07-28), skill
144
+ registration; domain backdraft.dev acquired 2026-07-28
145
+ - First external user (LoanCore validation layer vs Clarion eval substrate vs
146
+ open release feedback)
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Spencer Braun
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,253 @@
1
+ Metadata-Version: 2.4
2
+ Name: backdraft
3
+ Version: 0.2.0
4
+ Summary: Drop-in provenance for factual claims: one click from any claim to the evidence behind it.
5
+ License: MIT
6
+ License-File: LICENSE
7
+ Requires-Python: >=3.13
8
+ Requires-Dist: openpyxl>=3.1
9
+ Requires-Dist: pdfplumber>=0.11
10
+ Requires-Dist: typer>=0.12
11
+ Provides-Extra: entail
12
+ Requires-Dist: anthropic>=0.40; extra == 'entail'
13
+ Provides-Extra: vlm
14
+ Requires-Dist: openai>=1.0; extra == 'vlm'
15
+ Requires-Dist: pdf2image>=1.17; extra == 'vlm'
16
+ Description-Content-Type: text/markdown
17
+
18
+ <img src="assets/backdraft-mark.svg" width="56" alt="">
19
+
20
+ # backdraft
21
+
22
+ Drop-in provenance for factual claims: one click from any claim to the receipt
23
+ behind it.
24
+
25
+ An analyst who cannot show where a number came from either re-reads the source
26
+ or does not use the output. Backdraft removes that cost. In the usual loop, an
27
+ **agent** does the writing: it reads your source documents only through a gate
28
+ that mints a citation token over every span it shows, cites those tokens
29
+ inline, and a postprocess resolves them, attaches the verbatim evidence, and
30
+ renders one self-contained HTML file — the memo with the actual cited pages
31
+ and spreadsheet cells embedded inside it. The **human** just reads that file:
32
+ click a claim, see the source.
33
+
34
+ It works the same whether the writer is a model or a person, and whether the
35
+ citations were minted during writing or attached to a document afterwards.
36
+
37
+ Docs live at [backdraft.dev](https://backdraft.dev) — one page for people, and
38
+ [backdraft.dev/llms.txt](https://backdraft.dev/llms.txt) for agents.
39
+
40
+ ## 60 seconds
41
+
42
+ ```bash
43
+ uv tool install git+https://github.com/spencerbraun/backdraft
44
+ cd demo
45
+ ```
46
+
47
+ **Ingest** the sources. Every anchor is minted here — chunks for PDF pages,
48
+ cells for spreadsheet sheets.
49
+
50
+ ```console
51
+ $ backdraft init
52
+ registry: /…/demo/.backdraft
53
+ documents: 0
54
+
55
+ $ backdraft ingest sources/t12-summary.pdf sources/underwriting-model.xlsx
56
+ t12-summary t12-summary.pdf pdf 3 pages
57
+ underwriting-model underwriting-model.xlsx xlsx 2 pages
58
+ note: extracted with pdf-text (the embedded text layer). Glossy or scanned PDFs extract better through a vision model: install backdraft[vlm] and set BACKDRAFT_VLM_API_KEY in .backdraft/env.
59
+ ```
60
+
61
+ **Read.** The page arrives with a citable name over each chunk. This is the whole
62
+ mechanism: what you can cite is exactly what you were shown.
63
+
64
+ ```console
65
+ $ backdraft read t12-summary p1
66
+ # t12-summary p1 (page 1 of 3)
67
+
68
+ [bd:t12-summary:p1.c1:c2e8]
69
+ Bridgeview Commons — Trailing Twelve Month Summary
70
+
71
+ Property: Bridgeview Commons, 4400 Halsted Avenue, Columbus, OH 43214. 128 units across four three-story
72
+ garden buildings on 6.2 acres, built in 1998 and substantially renovated between 2019 and 2021. This summary
73
+ covers the trailing twelve months ended March 31, 2026, prepared from the borrower's monthly operating
74
+ statements and reconciled against the property manager's general ledger. Figures are unaudited.
75
+
76
+ [bd:t12-summary:p1.c2:7f11]
77
+ Total effective gross income for the trailing twelve months was $2,684,400, against gross potential rent of
78
+ $2,972,160. The resulting economic occupancy of 90.3% reflects an average physical occupancy of 94.1% offset
79
+ by concessions, bad debt, and vacancy loss. Physical occupancy was 96.4% in the most recent month and has not
80
+ fallen below 92.0% in any month of the period.
81
+
82
+ … five more chunks, c3 through c7.
83
+ ```
84
+
85
+ Chunks fall on the document's own paragraphs. A PDF stores glyphs at
86
+ coordinates, not paragraphs, so the extractor rebuilds the breaks from the line
87
+ geometry before the chunker sees the page.
88
+
89
+ Search results are citable too — no page read required to get an anchor:
90
+
91
+ ```console
92
+ $ backdraft search "24850000"
93
+ 1 result for "24850000"
94
+
95
+ [bd:underwriting-model:assumptions!B10:964a] underwriting-model p2
96
+ 24850000
97
+
98
+ [Read the page: backdraft read underwriting-model p2]
99
+ ```
100
+
101
+ A cell you can already see mints directly:
102
+
103
+ ```console
104
+ $ backdraft cell underwriting-model "assumptions!B10"
105
+ [bd:underwriting-model:assumptions!B10:964a] 24850000
106
+ ```
107
+
108
+ **Cite** by writing the token as the href of a markdown link on the claim span.
109
+ Multiple citations are `;`-separated in one href.
110
+
111
+ ```markdown
112
+ [net operating income of $1,429,600](bd:t12-summary:p1.c3:f10b)
113
+ [Effective gross income of $2,684,400](bd:t12-summary:p1.c2:7f11;bd:underwriting-model:rent-roll!B11:4b79)
114
+ ```
115
+
116
+ **Bind.** Every token resolves or is reported. Nothing drops silently.
117
+
118
+ ```console
119
+ $ backdraft bind memo.md --session s-bridgeview --check value-trace,overlap
120
+ bound 15 claim(s), 16 citation(s) [frontwalk]
121
+ resolved: 15
122
+ unresolved: 1
123
+ overlap: partial 4, pass 11
124
+ value-trace: pass 15
125
+ ! unresolved: bd:t12-summary:p4.c1:1a2b
126
+ wrote .backdraft/records/memo.backdraft.json
127
+
128
+ $ echo $?
129
+ 2
130
+ ```
131
+
132
+ Exit 2 means a citation did not resolve — the code a `Stop` hook or a CI job
133
+ gates on.
134
+
135
+ **Render.** One self-contained HTML file: the document, the receipts, the
136
+ evidence — cited page images, spreadsheet cells in place, the machine-readable
137
+ record — all embedded, no network, nothing to install for the reader.
138
+
139
+ ```console
140
+ $ backdraft render memo.md --to html
141
+ memo.backdraft.html
142
+ ```
143
+
144
+ That file is the whole deliverable: send it over email or Slack and the
145
+ recipient gets the full experience by double-clicking it.
146
+
147
+ The full version of this, with every command and every output:
148
+ [`demo/walkthrough.md`](demo/walkthrough.md). The artifact it produces is checked
149
+ in at [`demo/memo.backdraft.html`](demo/memo.backdraft.html).
150
+
151
+ ## The design in five sentences
152
+
153
+ Reading is a **gate**: source documents reach the writer only through `read` and
154
+ `search`, which mint a token over everything they show and record it in a
155
+ **ledger**, so the set of citable things is exactly the set of things shown — and
156
+ a citation to something the writer never saw is a distinguishable failure rather
157
+ than an invisible one. **Anchors are content-addressed**: a token's identity
158
+ derives from the document's content and a location inside it, and it binds to an
159
+ extraction snapshot, so re-ingesting the same bytes yields the same tokens and a
160
+ changed source yields `drifted` instead of a broken link. **The receipt travels
161
+ with the claim** — an anchor is not a pointer but a pointer plus the verbatim
162
+ snippet and its hash, which is why the finished artifact is defensible with the
163
+ registry deleted and the sources gone. **Verification is a set of switches,
164
+ default off**: resolution is inherent to binding, but `value-trace`, `overlap`
165
+ and `entail` are opt-in, recorded as graded evidence and never used as gates, so
166
+ out of the box this is provenance rather than a truth oracle. **Artifacts are
167
+ self-describing**: each carries a `$format` string matched byte-for-byte and an
168
+ embedded `$legend` that teaches a reader who has never heard of backdraft how to
169
+ decode and check it, so the format outlives this implementation.
170
+
171
+ Failures are data throughout: `unresolved`, `not_shown`, `drifted`, `malformed`
172
+ and `unmatched` are first-class records in the report and visible sections in the
173
+ artifact. Nothing is ever warned about and dropped.
174
+
175
+ ## What lives where
176
+
177
+ A working directory stays clean: the authored document and its artifact are
178
+ the only visible outputs. Everything else — registry, credentials, bind
179
+ records — lives under one hidden `.backdraft/` directory, and
180
+ `backdraft clean` tidies strays from older versions. The registry contains the
181
+ full text of everything ingested, so **gitignore `.backdraft/` for
182
+ confidential corpora**.
183
+
184
+ ## Install
185
+
186
+ Not yet on PyPI — install from the repository:
187
+
188
+ ```bash
189
+ uv tool install git+https://github.com/spencerbraun/backdraft # CLI
190
+ uv add "backdraft @ git+https://github.com/spencerbraun/backdraft" # as a dependency
191
+ pip install git+https://github.com/spencerbraun/backdraft # or plain pip
192
+ ```
193
+
194
+ Python 3.13+. Two extras:
195
+
196
+ | Extra | Adds | For |
197
+ |---|---|---|
198
+ | `[vlm]` | `openai`, `pdf2image` | the VLM extractor — the recommended path for real PDFs (glossy layouts, info boxes, scans): a vision model reads each page and its clean representation becomes the receipt. `--extractor auto` prefers it only on explicit, backdraft-scoped consent: `BACKDRAFT_VLM_API_KEY` (env or `.backdraft/env` — `backdraft init` writes a template). Ambient `OPENAI_API_KEY`/`OPENROUTER_API_KEY` are deliberately never read; a generic key in the environment is not consent to send documents to its provider. The default provider is OpenRouter running Gemini 3.1 Flash Lite; direct OpenAI is base_url + model, set explicitly. Without a scoped key, `auto` falls back to the keyless text layer and says so. Needs poppler installed separately |
199
+ | `[entail]` | `anthropic` | `bind --check entail`, the model-judge verifier |
200
+
201
+ ```bash
202
+ pip install "backdraft[vlm,entail] @ git+https://github.com/spencerbraun/backdraft"
203
+ ```
204
+
205
+ From a checkout:
206
+
207
+ ```bash
208
+ uv sync
209
+ uv run backdraft --help
210
+ uv run pytest
211
+ ```
212
+
213
+ ## Skills
214
+
215
+ Three agent skills ship in [`skills/`](skills). Copy the ones you want into your
216
+ agent's skills directory — for Claude Code, `~/.claude/skills/` (personal) or
217
+ `.claude/skills/` (per project):
218
+
219
+ ```bash
220
+ cp -r skills/backdraft skills/backdraft-backfill skills/backdraft-artifact ~/.claude/skills/
221
+ ```
222
+
223
+ | Skill | For |
224
+ |---|---|
225
+ | [`backdraft`](skills/backdraft/SKILL.md) | writing a new document from sources, citing as it goes |
226
+ | [`backdraft-backfill`](skills/backdraft-backfill/SKILL.md) | attributing a document that already exists |
227
+ | [`backdraft-artifact`](skills/backdraft-artifact/SKILL.md) | reading someone else's artifact cold |
228
+
229
+ The CLI is the system; a skill is one page telling an agent to use it. Nothing in
230
+ the skills is required to use backdraft by hand.
231
+
232
+ ## Documentation
233
+
234
+ | Document | Is |
235
+ |---|---|
236
+ | [DESIGN.md](DESIGN.md) | why it is shaped this way — principles, architecture, decision log |
237
+ | [SPEC.md](SPEC.md) | the builders' contract — types, grammar, DDL, CLI surface, module boundaries |
238
+ | [spec/tokens.md](spec/tokens.md) | the citation token grammar, normatively |
239
+ | [spec/chunking.md](spec/chunking.md) | the deterministic chunker |
240
+ | [spec/artifact.md](spec/artifact.md) | the artifact format — sidecar payload, legend, HTML rules |
241
+ | [demo/walkthrough.md](demo/walkthrough.md) | the whole thing end to end, with real output |
242
+
243
+ The three files under `spec/` are the portable specification: another
244
+ implementation reads them and nothing else.
245
+
246
+ ## Status
247
+
248
+ v1. The registry format, the token grammar and the artifact format are pinned
249
+ by prose specs and golden-file tests. What is deliberately queued — exhibits,
250
+ Excel region maps, more formats, `bd:calc` derivations — is written down in
251
+ [ROADMAP.md](ROADMAP.md).
252
+
253
+ MIT licensed.