sensemaking 0.21.1 → 0.22.0

package/README.md

@@ -93,7 +93,7 @@ ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10
 |---|---|
 | `presets` | named bundles of `include`/`exclude` globs, `k` (result count), `signals` (which engines this scope searches with, `words`, `links`, `vectors`; every signal whose prerequisites hold, unless the preset lists them exhaustively), `where` (a standing SQL filter). A file is indexed if any preset includes it, embedded if a model is named and some covering preset's `signals` include `vectors`; `status` shows each preset's coverage. |
 | `embed` | the model vectors are built with. Naming one gives the tree vectors; omitting the block means none at all, whatever the presets say. `sense download` fetches it. |
-| `store` | backing store engine: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`. The first command that opens such a tree installs that engine's package on its own (`@duckdb/node-api`, a one-time native download of ~110 MB; `@tursodatabase/database`, much smaller). The same commands and table names run on all three; what does not port is FTS5 syntax. Under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix (`foo*`), boolean (`AND`/`OR`/`NOT`), `NEAR`, initial-token (`^`), and column-filter (`title:foo`) operators with a named error that says how to rephrase or set `store` to `sqlite`; bare words and quoted phrases work on all three. Raw `sql` is a per-store dialect: the tables are portable, but sqlite's FTS5 `MATCH`/`snippet()`/`bm25()` do not run under `duckdb` or `turso`, so saved queries written in FTS5 syntax are sqlite dialect, and `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register SQL functions. `sense watch` requires `sqlite`; on the other two it errors at start naming the fix. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. |
+| `store` | backing store engine: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`. The first command that opens such a tree installs that engine's package on its own (`@duckdb/node-api`, a one-time native download of ~110 MB; `@tursodatabase/database`, much smaller). The same commands and table names run on all three; what does not port is FTS5 syntax. Under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix (`foo*`), boolean (`AND`/`OR`/`NOT`), `NEAR`, initial-token (`^`), and column-filter (`title:foo`) operators with a named error that says how to rephrase or set `store` to `sqlite`; bare words and quoted phrases work on all three. Raw `sql` is a per-store dialect: the tables are portable, but sqlite's FTS5 `MATCH`/`snippet()`/`bm25()` do not run under `duckdb` or `turso`, so saved queries written in FTS5 syntax are sqlite dialect, and `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register SQL functions. `sense watch` runs on all three; on `duckdb` and `turso`, which lock the cache file per connection, a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. |
 | `queries` | entries runnable as `sense <name>`, each naming the verb it runs: `{ sql }` for SQL (`?` binds positional args) or `{ search }` for a ranked search with its settings baked in, so `sense hot` needs no flags. Running an entry validates it: a typo'd column errors and exits nonzero, and a parameterised entry validates with any argument, since preparing precedes binding. |
 | `version` | schema version; older configs auto-migrate on load, noted on stderr. |
 

package/dist/cjs/chunk/group.js

@@ -144,13 +144,17 @@ function finalize(parts) {
     };
 }
 var NEWLINE_TOKENS = (0, _tokensts.estimateTokens)('\n');
-var SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'sentence'
-});
-var WORD_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'word'
-});
-function segmentsOf(text, segmenter) {
+// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever
+// split, so no command pays for a segmenter it never reaches.
+var SEGMENTERS = new Map();
+function segmentsOf(text, granularity) {
+    var segmenter = SEGMENTERS.get(granularity);
+    if (!segmenter) {
+        segmenter = new Intl.Segmenter(undefined, {
+            granularity: granularity
+        });
+        SEGMENTERS.set(granularity, segmenter);
+    }
     return Array.from(segmenter.segment(text), function(s) {
         return s.segment;
     });
@@ -194,7 +198,7 @@ function pack(segments, working) {
 // Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then
 // word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.
 function splitLineText(text, working) {
-    var sentences = segmentsOf(text, SENTENCE_SEGMENTER);
+    var sentences = segmentsOf(text, 'sentence');
     var out = [];
     var current = '';
     var tokens = 0;
@@ -213,7 +217,7 @@ function splitLineText(text, working) {
             if (sentenceTokens > working) {
                 var _out;
                 flush();
-                (_out = out).push.apply(_out, _to_consumable_array(pack(segmentsOf(sentence, WORD_SEGMENTER), working)));
+                (_out = out).push.apply(_out, _to_consumable_array(pack(segmentsOf(sentence, 'word'), working)));
                 continue;
             }
             if (current.length > 0 && tokens + sentenceTokens > working) flush();

package/dist/cjs/chunk/group.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","DEFAULT_TARGET_TOKENS","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","estimateTokens","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;+BA+NgBA;;;eAAAA;;;yBA/NY;uBACN;wBACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGtD,IAAMC,iBAAiB;AACvB,IAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIC,+BAAqB;QACzDC,IAAI,WAAEH,iBAAAA,2BAAAA,KAAMG,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAEjB,IAAI;WAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBC,IAAAA,wBAAc,EAAC;AACtC,IAAMC,qBAAqB,IAAIC,KAAKC,SAAS,CAACd,WAAW;IAAEe,aAAa;AAAW;AACnF,IAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACd,WAAW;IAAEe,aAAa;AAAO;AAE3E,SAASE,WAAW5B,IAAY,EAAE6B,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAAChC,OAAO,SAACiC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAIjC,UAAU;IACd,IAAIkC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBjB,IAAAA,wBAAc,EAACU;YACrC,IAAI5B,QAAQG,MAAM,GAAG,KAAK+B,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO7B,IAAI,CAACJ;gBACZA,UAAU;gBACVkC,SAAS;YACX;YACAlC,WAAW4B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAInC,QAAQG,MAAM,GAAG,GAAG8B,OAAO7B,IAAI,CAACJ;IACpC,OAAOiC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAcxC,IAAY,EAAEoC,OAAe;IAClD,IAAMK,YAAYb,WAAW5B,MAAMuB;IACnC,IAAMmB,MAAgB,EAAE;IACxB,IAAItC,UAAU;IACd,IAAIkC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAIvC,QAAQG,MAAM,GAAG,GAAG;YACtBmC,IAAIlC,IAAI,CAACJ;YACTA,UAAU;YACVkC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiBvB,IAAAA,wBAAc,EAACsB;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIlC,IAAI,OAARkC,MAAS,qBAAGR,KAAKN,WAAWgB,UAAUjB,iBAAiBS;gBACvD;YACF;YACA,IAAIhC,QAAQG,MAAM,GAAG,KAAK+B,SAASO,iBAAiBT,SAASO;YAC7DvC,WAAWwC;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAEnC,SAAiB,EAAEC,OAAe,EAAEmC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMnD,OACJ8C,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW/B,IAAI,CAAC,QAChBmC,IAAAA,cAAK,EAACJ,WAAW/B,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACsC;eAAMC,IAAAA,sBAAW,EAACD,EAAEE,IAAI;OAC7BtC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAASf,MAAAA;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAASyD,cAAcR,UAAoB,EAAEnC,SAAiB,EAAEC,OAAe,EAAEqB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMlC,IAAI+B,MAAMC,YAAYnC,WAAWC,SAASmC,WAAWC;IAC3D,IAAIF,WAAW1C,MAAM,KAAK,KAAKe,IAAAA,wBAAc,EAACL,EAAEjB,IAAI,IAAIoC,SAAS;QAC/D,OAAOI,cAAcvB,EAAEjB,IAAI,EAAEoC,SAASpB,GAAG,CAAC,SAAChB;mBAAU;gBAAEc,WAAAA;gBAAWC,SAAAA;gBAASf,MAAAA;gBAAMmB,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAASyC,mBAAmBC,KAAe,EAAE7C,SAAiB,EAAEC,OAAe,EAAEqB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAES,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIZ,aAAuB,EAAE;IAC7B,IAAIa,aAAahD;IACjB,IAAIwB,SAASsB;IACb,IAAK,IAAIG,OAAOjD,WAAWiD,QAAQhD,SAASgD,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMhB,WAAW1C,MAAM,GAAG,KAAK+B,SAAS,IAAIjB,iBAAiB;QACnE,IAAM6C,aAAa5C,IAAAA,wBAAc,EAAC0C;QAClC,IAAIf,WAAW1C,MAAM,GAAG,KAAK+B,SAAS2B,MAAMC,aAAa9B,SAAS;gBAChEyB;YAAAA,CAAAA,WAAAA,QAAOrD,IAAI,OAAXqD,UAAY,qBAAGJ,cAAcR,YAAYa,YAAYC,OAAO,GAAG3B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTwB,aAAaC;YACbd,WAAWzC,IAAI,CAACwD;YAChB1B,UAAU4B;YACV;QACF;QACAjB,WAAWzC,IAAI,CAACwD;QAChB1B,UAAU2B,MAAMC;IAClB;IACA,IAAIjB,WAAW1C,MAAM,GAAG,GAAGsD,CAAAA,UAAAA,QAAOrD,IAAI,OAAXqD,SAAY,qBAAGJ,cAAcR,YAAYa,YAAY/C,SAASqB,SAASc,WAAWC;IAC7G,OAAOU;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMjC,UAAUiC,SAASvE,YAAY;IACrC,IAAMwE,UAAUlC,UAAUzC;IAC1B,IAAM4E,WAA4C,EAAE;IACpD,IAAI7D,QAAgB,EAAE;IACtB,IAAI8D,iBAAiB;IACrB,IAAIlC,SAAS;IAEb,SAASmC;QACP,IAAMhF,QAAQgB,SAASC;QACvB,IAAIjB,OAAO8E,SAAS/D,IAAI,CAACf;QACzBiB,QAAQ,EAAE;QACV8D,iBAAiB;QACjBlC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASoC,QAAQ1E,IAAY,EAAE2E,QAAgB,EAAE7D,SAAiB,EAAEC,OAAe;QACjFuB,UAAU,AAAC5B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKC,IAAAA,wBAAc,EAACqD;QACnEjE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAASf,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAeoE,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAM/D,QAAN;YACH,IAAMuE,MAAMjB,MAAMkB,KAAK,CAACxE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM4D,cAAcxD,IAAAA,wBAAc,EAACsD;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOlD,MAAMH,MAAM,GAAG,IAAI+B,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWoB,mBAAmBC,OAAOtD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEqB,SAAS/B,MAAMC,IAAI,EAAE+D,SAASrE,IAAI,EAAE4D,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM3C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACXwD;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYxB,IAAAA,sBAAW,EAAClD,MAAMmD,IAAI;YACxC,IAAMmB,WAAWN,SAASrE,IAAI,KAAK,QAAQ4E,MAAMG;YAEjD,IAAI1E,MAAMC,IAAI,KAAK,WAAW;gBAC5BoE,QAAQK,WAAWJ,UAAUtE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMiE,eAAetE,MAAMH,MAAM,GAAG,KAAK+B,SAASjB,iBAAiByD,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUtE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3DyD;YAEA,IAAIA,kBAAkB9E,gBAAgB+E;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAAS9E,MAAMS,MAAe,EAAE+E,IAAY,EAAEpF,IAAmB;QAI7BqF;IAHzC,IAAMb,WAAWzE,eAAeC;IAChC,IAAM8D,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAejF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMkF,QAAN;YAAoCF,CAAAA,UAAAA,QAAO1E,IAAI,OAAX0E,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,qHAAqH;IACrH,IAAMgB,SACJhB,SAASrE,IAAI,KAAK,QACdkF,OAAOlE,GAAG,CAAC,SAACsE;eACVA,EAAEnE,KAAK,GACHmE,IACA,wCACKA;YACHtF,MAAM2D,MACHkB,KAAK,CAACS,EAAExE,SAAS,GAAG,GAAGwE,EAAEvE,OAAO,EAChCG,IAAI,CAAC,MACLqE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAEtF,IAAI,CAACuF,IAAI,GAAGhF,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACsE;eAAO;YAAExE,WAAWwE,EAAExE,SAAS;YAAEC,SAASuE,EAAEvE,OAAO;YAAEf,MAAMsF,EAAEtF,IAAI;QAAC;;AAC/H"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\n// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever\n// split, so no command pays for a segmenter it never reaches.\nconst SEGMENTERS = new Map<string, Intl.Segmenter>();\n\nfunction segmentsOf(text: string, granularity: 'sentence' | 'word'): string[] {\n  let segmenter = SEGMENTERS.get(granularity);\n  if (!segmenter) {\n    segmenter = new Intl.Segmenter(undefined, { granularity });\n    SEGMENTERS.set(granularity, segmenter);\n  }\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, 'sentence');\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, 'word'), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","DEFAULT_TARGET_TOKENS","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","estimateTokens","SEGMENTERS","Map","segmentsOf","granularity","segmenter","get","Intl","Segmenter","set","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;+BAqOgBA;;;eAAAA;;;yBArOY;uBACN;wBACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGtD,IAAMC,iBAAiB;AACvB,IAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIC,+BAAqB;QACzDC,IAAI,WAAEH,iBAAAA,2BAAAA,KAAMG,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAEjB,IAAI;WAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBC,IAAAA,wBAAc,EAAC;AACtC,gGAAgG;AAChG,8DAA8D;AAC9D,IAAMC,aAAa,IAAIC;AAEvB,SAASC,WAAWzB,IAAY,EAAE0B,WAAgC;IAChE,IAAIC,YAAYJ,WAAWK,GAAG,CAACF;IAC/B,IAAI,CAACC,WAAW;QACdA,YAAY,IAAIE,KAAKC,SAAS,CAACnB,WAAW;YAAEe,aAAAA;QAAY;QACxDH,WAAWQ,GAAG,CAACL,aAAaC;IAC9B;IACA,OAAOK,MAAMC,IAAI,CAACN,UAAUO,OAAO,CAAClC,OAAO,SAACmC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAInC,UAAU;IACd,IAAIoC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBnB,IAAAA,wBAAc,EAACY;YACrC,IAAI9B,QAAQG,MAAM,GAAG,KAAKiC,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO/B,IAAI,CAACJ;gBACZA,UAAU;gBACVoC,SAAS;YACX;YACApC,WAAW8B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAIrC,QAAQG,MAAM,GAAG,GAAGgC,OAAO/B,IAAI,CAACJ;IACpC,OAAOmC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAc1C,IAAY,EAAEsC,OAAe;IAClD,IAAMK,YAAYlB,WAAWzB,MAAM;IACnC,IAAM4C,MAAgB,EAAE;IACxB,IAAIxC,UAAU;IACd,IAAIoC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAIzC,QAAQG,MAAM,GAAG,GAAG;YACtBqC,IAAIpC,IAAI,CAACJ;YACTA,UAAU;YACVoC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiBzB,IAAAA,wBAAc,EAACwB;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIpC,IAAI,OAARoC,MAAS,qBAAGR,KAAKX,WAAWqB,UAAU,SAASR;gBAC/C;YACF;YACA,IAAIlC,QAAQG,MAAM,GAAG,KAAKiC,SAASO,iBAAiBT,SAASO;YAC7DzC,WAAW0C;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEqC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMrD,OACJgD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAWjC,IAAI,CAAC,QAChBqC,IAAAA,cAAK,EAACJ,WAAWjC,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACwC;eAAMC,IAAAA,sBAAW,EAACD,EAAEE,IAAI;OAC7BxC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAASf,MAAAA;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAAS2D,cAAcR,UAAoB,EAAErC,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMpC,IAAIiC,MAAMC,YAAYrC,WAAWC,SAASqC,WAAWC;IAC3D,IAAIF,WAAW5C,MAAM,KAAK,KAAKe,IAAAA,wBAAc,EAACL,EAAEjB,IAAI,IAAIsC,SAAS;QAC/D,OAAOI,cAAczB,EAAEjB,IAAI,EAAEsC,SAAStB,GAAG,CAAC,SAAChB;mBAAU;gBAAEc,WAAAA;gBAAWC,SAAAA;gBAASf,MAAAA;gBAAMmB,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAAS2C,mBAAmBC,KAAe,EAAE/C,SAAiB,EAAEC,OAAe,EAAEuB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAES,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIZ,aAAuB,EAAE;IAC7B,IAAIa,aAAalD;IACjB,IAAI0B,SAASsB;IACb,IAAK,IAAIG,OAAOnD,WAAWmD,QAAQlD,SAASkD,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMhB,WAAW5C,MAAM,GAAG,KAAKiC,SAAS,IAAInB,iBAAiB;QACnE,IAAM+C,aAAa9C,IAAAA,wBAAc,EAAC4C;QAClC,IAAIf,WAAW5C,MAAM,GAAG,KAAKiC,SAAS2B,MAAMC,aAAa9B,SAAS;gBAChEyB;YAAAA,CAAAA,WAAAA,QAAOvD,IAAI,OAAXuD,UAAY,qBAAGJ,cAAcR,YAAYa,YAAYC,OAAO,GAAG3B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTwB,aAAaC;YACbd,WAAW3C,IAAI,CAAC0D;YAChB1B,UAAU4B;YACV;QACF;QACAjB,WAAW3C,IAAI,CAAC0D;QAChB1B,UAAU2B,MAAMC;IAClB;IACA,IAAIjB,WAAW5C,MAAM,GAAG,GAAGwD,CAAAA,UAAAA,QAAOvD,IAAI,OAAXuD,SAAY,qBAAGJ,cAAcR,YAAYa,YAAYjD,SAASuB,SAASc,WAAWC;IAC7G,OAAOU;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMjC,UAAUiC,SAASzE,YAAY;IACrC,IAAM0E,UAAUlC,UAAU3C;IAC1B,IAAM8E,WAA4C,EAAE;IACpD,IAAI/D,QAAgB,EAAE;IACtB,IAAIgE,iBAAiB;IACrB,IAAIlC,SAAS;IAEb,SAASmC;QACP,IAAMlF,QAAQgB,SAASC;QACvB,IAAIjB,OAAOgF,SAASjE,IAAI,CAACf;QACzBiB,QAAQ,EAAE;QACVgE,iBAAiB;QACjBlC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASoC,QAAQ5E,IAAY,EAAE6E,QAAgB,EAAE/D,SAAiB,EAAEC,OAAe;QACjFyB,UAAU,AAAC9B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKC,IAAAA,wBAAc,EAACuD;QACnEnE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAASf,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAesE,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAMjE,QAAN;YACH,IAAMyE,MAAMjB,MAAMkB,KAAK,CAAC1E,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM8D,cAAc1D,IAAAA,wBAAc,EAACwD;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOpD,MAAMH,MAAM,GAAG,IAAIiC,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWoB,mBAAmBC,OAAOxD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEuB,SAASjC,MAAMC,IAAI,EAAEiE,SAASvE,IAAI,EAAE8D,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM7C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACX0D;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYxB,IAAAA,sBAAW,EAACpD,MAAMqD,IAAI;YACxC,IAAMmB,WAAWN,SAASvE,IAAI,KAAK,QAAQ8E,MAAMG;YAEjD,IAAI5E,MAAMC,IAAI,KAAK,WAAW;gBAC5BsE,QAAQK,WAAWJ,UAAUxE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMmE,eAAexE,MAAMH,MAAM,GAAG,KAAKiC,SAASnB,iBAAiB2D,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUxE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D2D;YAEA,IAAIA,kBAAkBhF,gBAAgBiF;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAAShF,MAAMS,MAAe,EAAEiF,IAAY,EAAEtF,IAAmB;QAI7BuF;IAHzC,IAAMb,WAAW3E,eAAeC;IAChC,IAAMgE,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAenF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMoF,QAAN;YAAoCF,CAAAA,UAAAA,QAAO5E,IAAI,OAAX4E,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,qHAAqH;IACrH,IAAMgB,SACJhB,SAASvE,IAAI,KAAK,QACdoF,OAAOpE,GAAG,CAAC,SAACwE;eACVA,EAAErE,KAAK,GACHqE,IACA,wCACKA;YACHxF,MAAM6D,MACHkB,KAAK,CAACS,EAAE1E,SAAS,GAAG,GAAG0E,EAAEzE,OAAO,EAChCG,IAAI,CAAC,MACLuE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAExF,IAAI,CAACyF,IAAI,GAAGlF,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACwE;eAAO;YAAE1E,WAAW0E,EAAE1E,SAAS;YAAEC,SAASyE,EAAEzE,OAAO;YAAEf,MAAMwF,EAAExF,IAAI;QAAC;;AAC/H"}

package/dist/cjs/commands/search.js

@@ -202,7 +202,7 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.
+// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md). Past this bound, rows get the JS excerpt.
 var EXCERPT_WINDOW = 160;
 // Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt
 // scan matches the same words the query matched on, not FTS5 syntax.

package/dist/cjs/commands/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // Raw substring scan, not porter-stemmed: a doc matched only through a stemmed variant\n    // finds no occurrence here, so fall back to the doc's start, unmarked.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n  const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n  const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = await rawScope(store, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n  // --where applies inside the candidate query (a post-filter would drop matches ranked past\n  // the pool) and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // Filtered before LIMIT, not after, or scoped notes ranking below the global top-`fetch`\n  // never reach the filter; joined against a temp table since real scopes exceed SQLITE_MAX_VARIABLE_NUMBER.\n  if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: LexicalHit[] = [];\n  // A query that is only whitespace has no searchable words: zero lexical rows, and the other\n  // signals compose normally.\n  if (signals.words !== undefined && terms.trim() !== '') {\n    try {\n      matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  await store.exec('DROP TABLE IF EXISTS _search');\n  // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n  // rounded score/similarity are printed at full precision in rows.\n  await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, hit TEXT, lines TEXT, similarity DOUBLE)');\n  if (candidates.size > 0) {\n    await store.runBatch(\n      'INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)',\n      [...candidates].map(([path, c]) => [path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n    );\n  }\n\n  // Reapplies --where even though _search is already scope+where filtered, since the join to\n  // frontmatter is already needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines: semantic rows carry their chunk's range, oversized-doc lexical rows gain one\n  // below, everything else stays null; similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const selectStmt = await store.prepare(\n    `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n  );\n  const rows = (await selectStmt.all(k)) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["search","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","row","prepare","get","start_line","end_line","cfg","opts","effective","k","signals","allPaths","scopePaths","scopeActive","allowedPaths","err","fetch","wantsVectors","e","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","matchRows","hits","oversized","chunkLines","chunkSimilarity","where","similarityCol","selectStmt","rows","bareTerms","resolveSearch","all","r","rawScope","narrowByWhere","searchError","max","vectors","undefined","embedConfig","localModelMissing","SenseError","presetName","MODEL_FILENAMES","model","scopeHasEmbeddings","materializeScope","Map","words","wordsCandidates","hit","links","linksCandidates","vectorsCandidates","exec","runBatch","c","score","via","has","readFileSync","join","baseDir","lines","featureEnabled"],"mappings":";;;;+BAwHsBA;;;eAAAA;;;sBAxHO;wBACR;uBAEsC;uBACR;wBACxB;6BAEC;uBAEkD;yBACV;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEpE,iFAAiF;AACjF,mGAAmG;AACnG,IAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,IAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,SAACC;eAAQA,IAAIH,OAAO,CAAC,kBAAkB;OAAK,iCAAiC;KACjFE,GAAG,CAAC,SAACC;eAAQA,IAAIC,WAAW,GAAGC,IAAI;OACnCC,MAAM,CAAC,SAACH;eAAQA,IAAII,MAAM,GAAG;;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,IAAMC,MAA2D,EAAE;QAC9D,kCAAA,2BAAA;;QAAL,QAAK,YAAcD,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAME,OAAN;YACH,IAAIC,MAAM;YACV,OAAS;gBACP,IAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;gBAC1C,IAAIC,UAAU,CAAC,GAAG;gBAClBH,IAAIK,IAAI,CAAC;oBAAEC,OAAOH;oBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;oBAAEK,MAAAA;gBAAK;gBACxDC,MAAMC,QAAQF,KAAKL,MAAM;YAC3B;QACF;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IASL,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,SAACC,GAAGC;eAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,IAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,IAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,IAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,uFAAuF;QACvF,uEAAuE;QACvE,IAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,AAAC,GAAmDrB,OAAjDkB,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,IAAkC,OAA7Ba,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;YAAMkC,QAAQ;QAAE;IAClH;IACA,IAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,IAAMsB,OAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAWN,wBAAX,SAAA,6BAAA,QAAA,yBAAA,iCAAgB;YAAhB,IAAMiC,IAAN;YACH,IAAIA,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,MAAK;YACpC,sFAAsF;YACtF,gFAAgF;YAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;YACtBD,OAAO,AAAC,GAAiCN,OAA/BA,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,GAAE,QAA8B,OAA3BmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,GAAE;YACpEyB,SAASC,EAAE1B,GAAG;QAChB;;QAPK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQLwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,IAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,IAAM6B,SAAS5B,OAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,AAAC,GAAWG,OAATG,QAAoCC,OAA3BJ,IAAI1C,OAAO,CAAC,QAAQ,MAAc,OAAP8C;QAAUL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;;YAC5DK,MACAC;;;;oBADO;;wBAAMH,MAAMI,OAAO,CAAC;;;oBAA3BF,OAAO;oBACA;;wBAAMA,KAAKG,GAAG,CAACJ,MAAMJ,MAAMA;;;oBAAlCM,MAAO;oBACb;;wBAAOA,MAAM,AAAC,IAAqBA,OAAlBA,IAAIG,UAAU,EAAC,KAAgB,OAAbH,IAAII,QAAQ,IAAK;;;;IACtD;;AAaO,SAAe/D;wCAAOwD,KAAY,EAAEQ,GAAmB,EAAEjD,KAAa;YAAEkD,MACvEC,WACEC,GAAGC,SAELC,UACAC,YACAC,aAGFC,cAGKC,MAGHC,OAIAC,cAEEC,GAKFC,uBAIAC,OACAC,WACAC,WAIAC,WAEAC,YACFC,WAMOV,MAILW,MAGAC,WAIFC,YACAC,sBAkBEC,OAGAC,eACAC,YAKAC,MAGEC,WACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMjC,KAELlB,MAMwBD,iBAApBI,SAASE;;;;;oBAhGwDmB,OAAAA,oEAAsB,CAAC;oBAC9FC,YAAY2B,IAAAA,sBAAa,EAAC7B,KAAKC;oBAC7BE,IAAeD,UAAfC,GAAGC,UAAYF,UAAZE;oBAEe;;wBAAMZ,MAAMI,OAAO,CAAC;;;oBAA3B;;wBAAO,cAAuDkC,GAAG;;;oBAA9EzB,WAAW,AAAE,cAAiG9D,GAAG,CAAC,SAACwF;+BAAMA,EAAEtC,IAAI;;oBAClH;;wBAAMuC,IAAAA,iBAAQ,EAACxC,OAAOQ,KAAKC,MAAMI;;;oBAA9CC,aAAa;oBACbC,cAAcD,WAAW/B,IAAI,GAAG8B,SAASzD,MAAM;;;;;;;;;oBAKpC;;wBAAMqF,IAAAA,sBAAa,EAACzC,OAAOc,YAAYJ,UAAUsB,KAAK;;;oBAArEhB,eAAe;;;;;;oBACRC;oBACP,MAAMyB,IAAAA,0BAAW,EAACzB,MAAc1D,OAAOmD,UAAUsB,KAAK;;oBAElDd,QAAQhC,KAAKyD,GAAG,CAAChC,IAAI,GAAG;oBAE9B,oFAAoF;oBACpF,0FAA0F;oBACpFQ,eAAeP,QAAQgC,OAAO,KAAKC;oBACzC,IAAI1B,cAAc;wBACVC,IAAI0B,IAAAA,oBAAW,EAACtC,MAAM,oEAAoE;wBAChG,IAAIuC,IAAAA,0BAAiB,EAAC3B,IAAI;4BACxB,MAAM,IAAI4B,oBAAU,CAAC,uBAAuB,AAAC,WAAoF5B,OAA1EV,UAAUuC,UAAU,EAAC,uDAA4EC,OAAvB9B,EAAE+B,KAAK,EAAC,iBAA+B,OAAhBD,wBAAe,EAAC;wBAC1K;oBACF;2BACwB/B;+BAAAA;;;;oBAAiB;;wBAAMiC,IAAAA,2BAAkB,EAACpD,OAAOQ,KAAKQ;;;2BAArC;;;oBAAnCK;oBAEN,2FAA2F;oBAC3F,kEAAkE;oBAC5DC,QAAQZ,UAAUsB,KAAK;oBACvBT,YAAYD,QAAQ,kDAAkD;oBACtEE,YAAYF,QAAQ,AAAC,QAAa,OAANA,OAAM,OAAK;yBAGzCP,aAAAA;;;;oBAAa;;wBAAMsC,IAAAA,yBAAgB,EAACrD,OAAO,iBAAiBc;;;oBAA/C;;;oBACXW,YAAYV,cAAc,2DAA2D;oBAErFW,aAAa,IAAI4B;oBACnB3B;yBAGAf,CAAAA,QAAQ2C,KAAK,KAAKV,aAAatF,MAAML,IAAI,OAAO,EAAC,GAAjD0D;;;;;;;;;;;;oBAEY;;wBAAM4C,IAAAA,0BAAe,EAACxD,OAAO0B,YAAYnE,OAAOgE,WAAWC,WAAWC,WAAWP,OAAON,QAAQ2C,KAAK;;;oBAAjH5B,YAAY;;;;;;oBACLV;oBACP,MAAMyB,IAAAA,0BAAW,EAACzB,MAAc1D,OAAO+D;;oBAGrCM,OAAO,IAAI0B,IAAI3B,UAAU5E,GAAG,CAAC,SAACwF;+BAAM;4BAACA,EAAEtC,IAAI;4BAAEsC,EAAEkB,GAAG;yBAAC;;oBACzD,wFAAwF;oBACxF,4EAA4E;oBACtE5B,YAAY,IAAIlD,IAAIgD,UAAUxE,MAAM,CAAC,SAACoF;+BAAMA,EAAEkB,GAAG,KAAK;uBAAM1G,GAAG,CAAC,SAACwF;+BAAMA,EAAEtC,IAAI;;yBAE/EW,CAAAA,QAAQ8C,KAAK,KAAKb,SAAQ,GAA1BjC;;;;oBAA6B;;wBAAM+C,IAAAA,0BAAe,EAAC3D,OAAO0B,YAAYC,WAAWd,UAAUG,cAAcE,OAAON,QAAQ8C,KAAK;;;oBAAhG;;;oBAE7B5B,aAAa,IAAIwB;oBACjBvB,kBAAkB,IAAIuB;yBACtBjC,iBAAAA;;;;oBACiC;;wBAAMuC,IAAAA,4BAAiB,EAAC5D,OAAOQ,KAAKkB,YAAYnE,OAAO2D,OAAOF,cAAcJ,QAAQgC,OAAO;;;0BAA3F,eAAhCd,iBAAAA,YAAYC,sBAAAA;;;oBAGjB;;wBAAM/B,MAAM6D,IAAI,CAAC;;;oBAAjB;oBACA,8FAA8F;oBAC9F,kEAAkE;oBAClE;;wBAAM7D,MAAM6D,IAAI,CAAC;;;oBAAjB;yBACInC,CAAAA,WAAW3C,IAAI,GAAG,CAAA,GAAlB2C;;;;oBACF;;wBAAM1B,MAAM8D,QAAQ,CAClB,8FACA,AAAC,qBAAGpC,YAAY3E,GAAG,CAAC;qEAAEkD,kBAAM8D;gCAA8BnC,WAAwBE,iBAA8BC;mCAA7E;gCAAC9B;gCAAM8D,EAAEC,KAAK;gCAAED,EAAEE,GAAG;iCAAErC,YAAAA,KAAKvB,GAAG,CAACJ,mBAAT2B,uBAAAA,YAAkB;iCAAME,kBAAAA,WAAWzB,GAAG,CAACJ,mBAAf6B,6BAAAA,kBAAwB;iCAAMC,uBAAAA,gBAAgB1B,GAAG,CAACJ,mBAApB8B,kCAAAA,uBAA6B;6BAAK;;;;oBAFpJ;;;oBAMF,2FAA2F;oBAC3F,qDAAqD;oBAC/CC,QAAQV,QAAQ,AAAC,UAAe,OAANA,OAAM,OAAK;oBAC3C,sFAAsF;oBACtF,qEAAqE;oBAC/DW,gBAAgBZ,kBAAkB,yBAAyB;oBAC9C;;wBAAMrB,MAAMI,OAAO,CACpC,AAAC,qIAEI4B,OAFgIC,eAAc,gIAExI,OAAND,OAAM;;;oBAHPE,aAAa;oBAKL;;wBAAMA,WAAWI,GAAG,CAAC3B;;;oBAA7BwB,OAAQ;yBAEVN,CAAAA,UAAU9C,IAAI,GAAG,CAAA,GAAjB8C;;;;oBACIO,YAAY1F,iBAAiBa;oBAC9B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAa4E;;;2BAAb,6BAAA,QAAA;;;;oBAAMhC,MAAN;oBACH,IAAIA,IAAIsD,GAAG,KAAK,QAAQ,CAAC5B,UAAUqC,GAAG,CAAC/D,IAAIF,IAAI,GAAa;;;;oBACxDhB,OAAAA,KAAAA;oBACJ,IAAI;wBACFA,OAAOkF,IAAAA,oBAAY,EAACC,IAAAA,cAAI,EAAC5D,IAAI6D,OAAO,EAAElE,IAAIF,IAAI,GAAa;oBAC7D,EAAE,eAAM;wBACN;;;2BAAU,mEAAmE;oBAC/E;oBAC4BjB,kBAAAA,eAAeC,MAAMmD,YAAzChD,UAAoBJ,gBAApBI,SAASE,SAAWN,gBAAXM;oBACjBa,IAAIsD,GAAG,GAAGrE;yBACNe,CAAAA,IAAImE,KAAK,IAAI,IAAG,GAAhBnE;;;;yBAA+BoE,IAAAA,uBAAc,EAAC/D,KAAK,aAApB+D;;;;oBAAkC;;wBAAMxE,aAAaC,OAAOG,IAAIF,IAAI,EAAYL,aAAaX,MAAMK;;;4BAAjE;;;;;;4BAA4E;;;oBAA1Ha,IAAImE,KAAK;;;oBAV7B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAcP;;wBAAOnC;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // Raw substring scan, not porter-stemmed: a doc matched only through a stemmed variant\n    // finds no occurrence here, so fall back to the doc's start, unmarked.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n  const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n  const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = await rawScope(store, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n  // --where applies inside the candidate query (a post-filter would drop matches ranked past\n  // the pool) and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // Filtered before LIMIT, not after, or scoped notes ranking below the global top-`fetch`\n  // never reach the filter; joined against a temp table since real scopes exceed SQLITE_MAX_VARIABLE_NUMBER.\n  if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: LexicalHit[] = [];\n  // A query that is only whitespace has no searchable words: zero lexical rows, and the other\n  // signals compose normally.\n  if (signals.words !== undefined && terms.trim() !== '') {\n    try {\n      matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  await store.exec('DROP TABLE IF EXISTS _search');\n  // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n  // rounded score/similarity are printed at full precision in rows.\n  await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, hit TEXT, lines TEXT, similarity DOUBLE)');\n  if (candidates.size > 0) {\n    await store.runBatch(\n      'INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)',\n      [...candidates].map(([path, c]) => [path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n    );\n  }\n\n  // Reapplies --where even though _search is already scope+where filtered, since the join to\n  // frontmatter is already needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines: semantic rows carry their chunk's range, oversized-doc lexical rows gain one\n  // below, everything else stays null; similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const selectStmt = await store.prepare(\n    `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n  );\n  const rows = (await selectStmt.all(k)) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["search","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","row","prepare","get","start_line","end_line","cfg","opts","effective","k","signals","allPaths","scopePaths","scopeActive","allowedPaths","err","fetch","wantsVectors","e","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","matchRows","hits","oversized","chunkLines","chunkSimilarity","where","similarityCol","selectStmt","rows","bareTerms","resolveSearch","all","r","rawScope","narrowByWhere","searchError","max","vectors","undefined","embedConfig","localModelMissing","SenseError","presetName","MODEL_FILENAMES","model","scopeHasEmbeddings","materializeScope","Map","words","wordsCandidates","hit","links","linksCandidates","vectorsCandidates","exec","runBatch","c","score","via","has","readFileSync","join","baseDir","lines","featureEnabled"],"mappings":";;;;+BAwHsBA;;;eAAAA;;;sBAxHO;wBACR;uBAEsC;uBACR;wBACxB;6BAEC;uBAEkD;yBACV;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEpE,iFAAiF;AACjF,0GAA0G;AAC1G,IAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,IAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,SAACC;eAAQA,IAAIH,OAAO,CAAC,kBAAkB;OAAK,iCAAiC;KACjFE,GAAG,CAAC,SAACC;eAAQA,IAAIC,WAAW,GAAGC,IAAI;OACnCC,MAAM,CAAC,SAACH;eAAQA,IAAII,MAAM,GAAG;;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,IAAMC,MAA2D,EAAE;QAC9D,kCAAA,2BAAA;;QAAL,QAAK,YAAcD,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAME,OAAN;YACH,IAAIC,MAAM;YACV,OAAS;gBACP,IAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;gBAC1C,IAAIC,UAAU,CAAC,GAAG;gBAClBH,IAAIK,IAAI,CAAC;oBAAEC,OAAOH;oBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;oBAAEK,MAAAA;gBAAK;gBACxDC,MAAMC,QAAQF,KAAKL,MAAM;YAC3B;QACF;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IASL,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,SAACC,GAAGC;eAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,IAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,IAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,IAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,uFAAuF;QACvF,uEAAuE;QACvE,IAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,AAAC,GAAmDrB,OAAjDkB,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,IAAkC,OAA7Ba,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;YAAMkC,QAAQ;QAAE;IAClH;IACA,IAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,IAAMsB,OAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAWN,wBAAX,SAAA,6BAAA,QAAA,yBAAA,iCAAgB;YAAhB,IAAMiC,IAAN;YACH,IAAIA,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,MAAK;YACpC,sFAAsF;YACtF,gFAAgF;YAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;YACtBD,OAAO,AAAC,GAAiCN,OAA/BA,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,GAAE,QAA8B,OAA3BmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,GAAE;YACpEyB,SAASC,EAAE1B,GAAG;QAChB;;QAPK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQLwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,IAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,IAAM6B,SAAS5B,OAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,AAAC,GAAWG,OAATG,QAAoCC,OAA3BJ,IAAI1C,OAAO,CAAC,QAAQ,MAAc,OAAP8C;QAAUL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;;YAC5DK,MACAC;;;;oBADO;;wBAAMH,MAAMI,OAAO,CAAC;;;oBAA3BF,OAAO;oBACA;;wBAAMA,KAAKG,GAAG,CAACJ,MAAMJ,MAAMA;;;oBAAlCM,MAAO;oBACb;;wBAAOA,MAAM,AAAC,IAAqBA,OAAlBA,IAAIG,UAAU,EAAC,KAAgB,OAAbH,IAAII,QAAQ,IAAK;;;;IACtD;;AAaO,SAAe/D;wCAAOwD,KAAY,EAAEQ,GAAmB,EAAEjD,KAAa;YAAEkD,MACvEC,WACEC,GAAGC,SAELC,UACAC,YACAC,aAGFC,cAGKC,MAGHC,OAIAC,cAEEC,GAKFC,uBAIAC,OACAC,WACAC,WAIAC,WAEAC,YACFC,WAMOV,MAILW,MAGAC,WAIFC,YACAC,sBAkBEC,OAGAC,eACAC,YAKAC,MAGEC,WACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMjC,KAELlB,MAMwBD,iBAApBI,SAASE;;;;;oBAhGwDmB,OAAAA,oEAAsB,CAAC;oBAC9FC,YAAY2B,IAAAA,sBAAa,EAAC7B,KAAKC;oBAC7BE,IAAeD,UAAfC,GAAGC,UAAYF,UAAZE;oBAEe;;wBAAMZ,MAAMI,OAAO,CAAC;;;oBAA3B;;wBAAO,cAAuDkC,GAAG;;;oBAA9EzB,WAAW,AAAE,cAAiG9D,GAAG,CAAC,SAACwF;+BAAMA,EAAEtC,IAAI;;oBAClH;;wBAAMuC,IAAAA,iBAAQ,EAACxC,OAAOQ,KAAKC,MAAMI;;;oBAA9CC,aAAa;oBACbC,cAAcD,WAAW/B,IAAI,GAAG8B,SAASzD,MAAM;;;;;;;;;oBAKpC;;wBAAMqF,IAAAA,sBAAa,EAACzC,OAAOc,YAAYJ,UAAUsB,KAAK;;;oBAArEhB,eAAe;;;;;;oBACRC;oBACP,MAAMyB,IAAAA,0BAAW,EAACzB,MAAc1D,OAAOmD,UAAUsB,KAAK;;oBAElDd,QAAQhC,KAAKyD,GAAG,CAAChC,IAAI,GAAG;oBAE9B,oFAAoF;oBACpF,0FAA0F;oBACpFQ,eAAeP,QAAQgC,OAAO,KAAKC;oBACzC,IAAI1B,cAAc;wBACVC,IAAI0B,IAAAA,oBAAW,EAACtC,MAAM,oEAAoE;wBAChG,IAAIuC,IAAAA,0BAAiB,EAAC3B,IAAI;4BACxB,MAAM,IAAI4B,oBAAU,CAAC,uBAAuB,AAAC,WAAoF5B,OAA1EV,UAAUuC,UAAU,EAAC,uDAA4EC,OAAvB9B,EAAE+B,KAAK,EAAC,iBAA+B,OAAhBD,wBAAe,EAAC;wBAC1K;oBACF;2BACwB/B;+BAAAA;;;;oBAAiB;;wBAAMiC,IAAAA,2BAAkB,EAACpD,OAAOQ,KAAKQ;;;2BAArC;;;oBAAnCK;oBAEN,2FAA2F;oBAC3F,kEAAkE;oBAC5DC,QAAQZ,UAAUsB,KAAK;oBACvBT,YAAYD,QAAQ,kDAAkD;oBACtEE,YAAYF,QAAQ,AAAC,QAAa,OAANA,OAAM,OAAK;yBAGzCP,aAAAA;;;;oBAAa;;wBAAMsC,IAAAA,yBAAgB,EAACrD,OAAO,iBAAiBc;;;oBAA/C;;;oBACXW,YAAYV,cAAc,2DAA2D;oBAErFW,aAAa,IAAI4B;oBACnB3B;yBAGAf,CAAAA,QAAQ2C,KAAK,KAAKV,aAAatF,MAAML,IAAI,OAAO,EAAC,GAAjD0D;;;;;;;;;;;;oBAEY;;wBAAM4C,IAAAA,0BAAe,EAACxD,OAAO0B,YAAYnE,OAAOgE,WAAWC,WAAWC,WAAWP,OAAON,QAAQ2C,KAAK;;;oBAAjH5B,YAAY;;;;;;oBACLV;oBACP,MAAMyB,IAAAA,0BAAW,EAACzB,MAAc1D,OAAO+D;;oBAGrCM,OAAO,IAAI0B,IAAI3B,UAAU5E,GAAG,CAAC,SAACwF;+BAAM;4BAACA,EAAEtC,IAAI;4BAAEsC,EAAEkB,GAAG;yBAAC;;oBACzD,wFAAwF;oBACxF,4EAA4E;oBACtE5B,YAAY,IAAIlD,IAAIgD,UAAUxE,MAAM,CAAC,SAACoF;+BAAMA,EAAEkB,GAAG,KAAK;uBAAM1G,GAAG,CAAC,SAACwF;+BAAMA,EAAEtC,IAAI;;yBAE/EW,CAAAA,QAAQ8C,KAAK,KAAKb,SAAQ,GAA1BjC;;;;oBAA6B;;wBAAM+C,IAAAA,0BAAe,EAAC3D,OAAO0B,YAAYC,WAAWd,UAAUG,cAAcE,OAAON,QAAQ8C,KAAK;;;oBAAhG;;;oBAE7B5B,aAAa,IAAIwB;oBACjBvB,kBAAkB,IAAIuB;yBACtBjC,iBAAAA;;;;oBACiC;;wBAAMuC,IAAAA,4BAAiB,EAAC5D,OAAOQ,KAAKkB,YAAYnE,OAAO2D,OAAOF,cAAcJ,QAAQgC,OAAO;;;0BAA3F,eAAhCd,iBAAAA,YAAYC,sBAAAA;;;oBAGjB;;wBAAM/B,MAAM6D,IAAI,CAAC;;;oBAAjB;oBACA,8FAA8F;oBAC9F,kEAAkE;oBAClE;;wBAAM7D,MAAM6D,IAAI,CAAC;;;oBAAjB;yBACInC,CAAAA,WAAW3C,IAAI,GAAG,CAAA,GAAlB2C;;;;oBACF;;wBAAM1B,MAAM8D,QAAQ,CAClB,8FACA,AAAC,qBAAGpC,YAAY3E,GAAG,CAAC;qEAAEkD,kBAAM8D;gCAA8BnC,WAAwBE,iBAA8BC;mCAA7E;gCAAC9B;gCAAM8D,EAAEC,KAAK;gCAAED,EAAEE,GAAG;iCAAErC,YAAAA,KAAKvB,GAAG,CAACJ,mBAAT2B,uBAAAA,YAAkB;iCAAME,kBAAAA,WAAWzB,GAAG,CAACJ,mBAAf6B,6BAAAA,kBAAwB;iCAAMC,uBAAAA,gBAAgB1B,GAAG,CAACJ,mBAApB8B,kCAAAA,uBAA6B;6BAAK;;;;oBAFpJ;;;oBAMF,2FAA2F;oBAC3F,qDAAqD;oBAC/CC,QAAQV,QAAQ,AAAC,UAAe,OAANA,OAAM,OAAK;oBAC3C,sFAAsF;oBACtF,qEAAqE;oBAC/DW,gBAAgBZ,kBAAkB,yBAAyB;oBAC9C;;wBAAMrB,MAAMI,OAAO,CACpC,AAAC,qIAEI4B,OAFgIC,eAAc,gIAExI,OAAND,OAAM;;;oBAHPE,aAAa;oBAKL;;wBAAMA,WAAWI,GAAG,CAAC3B;;;oBAA7BwB,OAAQ;yBAEVN,CAAAA,UAAU9C,IAAI,GAAG,CAAA,GAAjB8C;;;;oBACIO,YAAY1F,iBAAiBa;oBAC9B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAa4E;;;2BAAb,6BAAA,QAAA;;;;oBAAMhC,MAAN;oBACH,IAAIA,IAAIsD,GAAG,KAAK,QAAQ,CAAC5B,UAAUqC,GAAG,CAAC/D,IAAIF,IAAI,GAAa;;;;oBACxDhB,OAAAA,KAAAA;oBACJ,IAAI;wBACFA,OAAOkF,IAAAA,oBAAY,EAACC,IAAAA,cAAI,EAAC5D,IAAI6D,OAAO,EAAElE,IAAIF,IAAI,GAAa;oBAC7D,EAAE,eAAM;wBACN;;;2BAAU,mEAAmE;oBAC/E;oBAC4BjB,kBAAAA,eAAeC,MAAMmD,YAAzChD,UAAoBJ,gBAApBI,SAASE,SAAWN,gBAAXM;oBACjBa,IAAIsD,GAAG,GAAGrE;yBACNe,CAAAA,IAAImE,KAAK,IAAI,IAAG,GAAhBnE;;;;yBAA+BoE,IAAAA,uBAAc,EAAC/D,KAAK,aAApB+D;;;;oBAAkC;;wBAAMxE,aAAaC,OAAOG,IAAIF,IAAI,EAAYL,aAAaX,MAAMK;;;4BAAjE;;;;;;4BAA4E;;;oBAA1Ha,IAAImE,KAAK;;;oBAV7B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAcP;;wBAAOnC;;;;IACT"}

package/dist/cjs/index.d.cts

@@ -6,7 +6,7 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult } from './store/index.js';
+export type { OpenResult, Stages } from './store/index.js';
 export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';

package/dist/cjs/index.d.ts

@@ -6,7 +6,7 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult } from './store/index.js';
+export type { OpenResult, Stages } from './store/index.js';
 export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';

package/dist/cjs/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAW9DA;eAAAA,0BAAgB;;QANhBC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIQC;eAAAA,oBAAU;;QANXC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMXC;eAAbC,mBAAS;;QAR9BC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEsC;uBAIvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult, Stages } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAW9DA;eAAAA,0BAAgB;;QANhBC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIQC;eAAAA,oBAAU;;QANXC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMXC;eAAbC,mBAAS;;QAR9BC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEsC;uBAIvC"}

package/dist/cjs/scan/pool.d.cts

@@ -5,6 +5,7 @@ import type { FileStat } from './list.js';
 export type FileResult = {
     doc: ParsedDoc;
     warnings: string[];
+    parseMs?: number;
 };
 export declare class ParsePool {
     private pool;

package/dist/cjs/scan/pool.d.ts

@@ -5,6 +5,7 @@ import type { FileStat } from './list.js';
 export type FileResult = {
     doc: ParsedDoc;
     warnings: string[];
+    parseMs?: number;
 };
 export declare class ParsePool {
     private pool;

package/dist/cjs/scan/pool.js

@@ -222,7 +222,8 @@ var ParsePool = /*#__PURE__*/ function() {
                                             2,
                                             {
                                                 doc: result.doc,
-                                                warnings: result.warnings
+                                                warnings: result.warnings,
+                                                parseMs: result.parseMs
                                             }
                                         ];
                                 }

package/dist/cjs/scan/pool.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\nexport type FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","close","undefined","destroy"],"mappings":";;;;+BAoCaA;;;eAAAA;;;0BApCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAMO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;4CAAC;;;;wBACtD;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEEvB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGwB;wBACZ;;4BAAMxB,KAAKyB,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW1C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":";;;;+BAqCaA;;;eAAAA;;;0BArCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAOO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;gDAAEC,SAASN,OAAOM,OAAO;4CAAC;;;;wBAC/E;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEExB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGyB;wBACZ;;4BAAMzB,KAAK0B,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW3C"}

package/dist/cjs/scan/reparse.d.cts

@@ -7,6 +7,7 @@ export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
     newColumns: string[];
+    workerParseMs: number;
 }
 export interface ReparseOptions {
     threshold?: number;

package/dist/cjs/scan/reparse.d.ts

@@ -7,6 +7,7 @@ export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
     newColumns: string[];
+    workerParseMs: number;
 }
 export interface ReparseOptions {
     threshold?: number;

package/dist/cjs/scan/reparse.js

@@ -265,7 +265,7 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
 }
 function reparseFiles(_0, _1, _2, _3, _4) {
     return _async_to_generator(function(files, features, cfg, knownColumns, onParsed) {
-        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, results, _tmp, docs, warnings, newColumns, seen, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
+        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, results, _tmp, docs, warnings, newColumns, seen, workerParseMs, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, parseMs, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
@@ -296,12 +296,14 @@ function reparseFiles(_0, _1, _2, _3, _4) {
                     warnings = [];
                     newColumns = [];
                     seen = new Set(knownColumns);
+                    workerParseMs = 0;
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
                         for(_iterator = results[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                            _step_value = _step.value, doc = _step_value.doc, fileWarnings = _step_value.warnings;
+                            _step_value = _step.value, doc = _step_value.doc, fileWarnings = _step_value.warnings, parseMs = _step_value.parseMs;
                             ;
                             (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
+                            workerParseMs += parseMs !== null && parseMs !== void 0 ? parseMs : 0;
                             _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                             try {
                                 for(_iterator1 = Object.keys(doc.data)[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
@@ -346,7 +348,8 @@ function reparseFiles(_0, _1, _2, _3, _4) {
                         {
                             docs: docs,
                             warnings: warnings,
-                            newColumns: newColumns
+                            newColumns: newColumns,
+                            workerParseMs: workerParseMs
                         }
                     ];
             }

package/dist/cjs/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","doc","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAoCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAtEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAC1BJ,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMK;;;;;oBAX+IR,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBS,KAAKC,GAAG,CAACnC,qBAAqBoC,IAAAA,4BAAoB;yBAC3E5B,CAAAA,MAAM6B,MAAM,IAAIX,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIQ,IAAId;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyCd,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQqB,kBAAAA,KAAeC,2BAAVJ;;4BAChBA,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGI;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACT,IAAIU,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACH,KAAKY,GAAG,CAACT,MAAM;wCAClBH,KAAKa,GAAG,CAACV;wCACTJ,WAAWhB,IAAI,CAACoB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLN,KAAKd,IAAI,CAACkB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEJ,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;wBAAW;;;;IACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n  // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n  workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let workerParseMs = 0;\n  for (const { doc, warnings: fileWarnings, parseMs } of results) {\n    warnings.push(...fileWarnings);\n    workerParseMs += parseMs ?? 0;\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns, workerParseMs };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","workerParseMs","doc","fileWarnings","parseMs","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAsCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAxEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACFC,eACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAAcC,SACxCN,WAEK,4BAAA,oBAAA,iBAAA,YAAA,QAAMO;;;;;oBAb+IV,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBW,KAAKC,GAAG,CAACrC,qBAAqBsC,IAAAA,4BAAoB;yBAC3E9B,CAAAA,MAAM+B,MAAM,IAAIb,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIU,IAAIhB;oBACjBO,gBAAgB;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAkDrB,8BAAlD,6BAAA,QAAA,yBAAA,iCAA2D;0CAA3D,aAAQsB,kBAAAA,KAAeC,2BAAVL,UAAwBM,sBAAAA;;4BACxCN,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGK;4BACjBF,iBAAiBG,oBAAAA,qBAAAA,UAAW;4BACvB,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACV,IAAIW,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACL,KAAKc,GAAG,CAACT,MAAM;wCAClBL,KAAKe,GAAG,CAACV;wCACTN,WAAWhB,IAAI,CAACsB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLR,KAAKd,IAAI,CAACmB;wBACZ;;wBAVK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL;;wBAAO;4BAAEL,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;4BAAYE,eAAAA;wBAAc;;;;IACrD"}

package/dist/cjs/store/builder.d.cts

@@ -1,11 +1,13 @@
 import type { Config } from '../config/index.js';
 import type { EmbedChangeKind } from './embed-scope.js';
 import type { FeatureToggle } from './feature-scope.js';
+import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
     build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
+        stages: Stages;
     }>;
     invalidate(kind: EmbedChangeKind): Promise<{
         parsed: number;

package/dist/cjs/store/builder.d.ts

@@ -1,11 +1,13 @@
 import type { Config } from '../config/index.js';
 import type { EmbedChangeKind } from './embed-scope.js';
 import type { FeatureToggle } from './feature-scope.js';
+import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
     build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
+        stages: Stages;
     }>;
     invalidate(kind: EmbedChangeKind): Promise<{
         parsed: number;

package/dist/cjs/store/builder.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","nullEmbedVectors","conn","dialect","withTransaction","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","existingPaths","files","docs","paths","delta","prepare","Set","all","map","r","path","listFiles","filter","f","embed","has","relPath","length","reparseFiles","undefined","d","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","feature","docsForFeature","NARROW_FEATURE_TABLE","name","FEATURES","find","Error","turnedOn","afterReconcile","nullRank","columns","getColumns","rerunRank","rank","invalidateFeatureToggles","toggles","byName","linksToggle","rankToggle","Map","get","ParsePool","build","forcedPaths","reconcile","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":";;;;+BA6JgBA;;;eAAAA;;;uBA5JM;uBACG;sBACJ;wBAEK;sBACA;yBACG;8BAGQ;2BACX;wBACC;6BACK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBhC,6FAA6F;AAC7F,wEAAwE;AACxE,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;;;;;oBACzE;;wBAAMC,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAAsDF,QAAQG,SAAS;;;oBAAnH;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;;IACrB;;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,SAAeC,kBAAkBN,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe;;YACnHC,cACAC,kBAGAC,OAGEC,MACFC,OACAC;;;;oBATe;;wBAAMf,KAAKgB,OAAO,CAAC;;;oBAAlCN,eAAe;wBACKO;oBAAK;;wBAAMP,aAAaQ,GAAG;;;oBAA/CP,gBAAgB,IAAA,CAAA,EAAA,MAAIM;;wBAAK,cAA0BE,GAAG,CAAC,SAACC;mCAAM,AAACA,EAAuBC,IAAI;;sBAAC;oBACjG,8FAA8F;oBAC9F,4FAA4F;oBACtFT,QAAQU,IAAAA,mBAAS,EAACf,KAAKC,SAASe,MAAM,CAAC,SAACC;+BAAMA,EAAEC,KAAK,IAAId,cAAce,GAAG,CAACF,EAAEG,OAAO;;oBAC1F,IAAIf,MAAMgB,MAAM,KAAK,GAAG;;wBAAO;4BAAEvB,QAAQ;wBAAE;;oBAE1B;;wBAAMwB,IAAAA,uBAAY,EAACjB;4BAAQa,cAAK;2BAAGlB,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAA9EI,OAAS,cAATA;oBACFC,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACjCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,KAAK;wBAAMC,QAAQ;oBAAK;oBAChF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQyB,eACAA;;;;4CADN;;iDAAMA,gBAAAA,cAAK,CAACU,MAAM,cAAZV,oCAAAA,mBAAAA,cAAK,EAAUzB,MAAMc,OAAOC;;;4CAAlC;4CACA;;iDAAMU,eAAAA,cAAK,CAACW,KAAK,cAAXX,mCAAAA,kBAAAA,cAAK,EACTzB,MACAa,KAAKM,GAAG,CAAC,SAACY;2DAAO;wDAAEV,MAAMU,EAAEJ,OAAO;wDAAEU,WAAWN,EAAEM,SAAS,CAACZ,KAAK;oDAAC;oDACjEV;;;4CAHF;;;;;;4BAKF;2BACAd,QAAQG,SAAS;;;oBAVnB;oBAYA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,SAAeU,uBAAuBtC,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE8B,MAAqB;;YAC/IC,OACAC,SAQA7B,OACEC,MACF6B,gBACA5B,OAIAC;;;;oBAhBAyB,QAAQG,oCAAoB,CAACJ,OAAOK,IAAI,CAAC;oBACzCH,UAAUI,iBAAQ,CAACC,IAAI,CAAC,SAACtB;+BAAMA,EAAEoB,IAAI,KAAKL,OAAOK,IAAI;;oBAC3D,IAAI,CAACH,SAAS,MAAM,IAAIM,MAAM,AAAC,wDAAmE,OAAZR,OAAOK,IAAI,EAAC;yBAE9F,CAACL,OAAOS,QAAQ,EAAhB;;;;oBACF;;wBAAM9C,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;2BAAUvC,QAAQG,SAAS;;;oBAAtF;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;oBAGfO,QAAQU,IAAAA,mBAAS,EAACf,KAAKC;oBACZ;;wBAAMqB,IAAAA,uBAAY,EAACjB;4BAAQ6B;2BAAUlC,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAAhFI,OAAS,cAATA;oBACF6B,iBAAiB7B,KAAKM,GAAG,CAAC,SAACY;+BAAO;4BAAEV,MAAMU,EAAEJ,OAAO;4BAAEU,WAAWN,EAAEM,SAAS,CAACI,QAAQG,IAAI,CAAC;wBAAC;;oBAC1F9B,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACvC,gGAAgG;oBAChG,8FAA8F;oBAC9F,8CAA8C;oBACxCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,OAAOnB;wBAAOoB,QAAQ;oBAAK;oBACnF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCAEQyC,gBACAA;;;;4CAFN;;gDAAMzC,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;;;4CAA/B;4CACA;;iDAAMC,iBAAAA,QAAQL,KAAK,cAAbK,qCAAAA,oBAAAA,SAAgBzC,MAAM0C,gBAAgB3B;;;4CAA5C;4CACA;;iDAAM0B,0BAAAA,QAAQQ,cAAc,cAAtBR,8CAAAA,6BAAAA,SAAyBzC,MAAMe;;;4CAArC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBAPnB;oBASA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,SAAesB,SAASlD,IAAgB,EAAEC,OAAyB;;YAC3DkD;;;;oBAAU;;wBAAMC,IAAAA,oBAAU,EAACpD;;;oBAA3BmD,UAAU;oBAChB,IAAI,CAACA,QAAQzB,GAAG,CAAC,UAAU;;;oBAC3B;;wBAAMxB,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAA0CF,QAAQG,SAAS;;;oBAAvG;;;;;;IACF;;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAAeiD,UAAUrD,IAAgB,EAAEC,OAAyB;;YAC5Dc;;;;oBAAAA,QAAwB;wBAAEH,KAAK;wBAAMoB,QAAQ;wBAAMC,KAAK;wBAAMC,QAAQ;oBAAK;oBACjF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQsD;;;;4CAAN;;iDAAMA,uBAAAA,YAAI,CAACL,cAAc,cAAnBK,2CAAAA,0BAAAA,YAAI,EAAkBtD,MAAMe;;;4CAAlC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBALnB;;;;;;IAOF;;AAEA,SAAemD,yBAAyBvD,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE+C,OAAwB;;YACpJC,QACFpD,QAIEqD,aAWAC,YAMD,2BAAA,mBAAA,gBAAA,WAAA,OAAMpB;;;;oBAtBLkB,SAAS,IAAIG,IAAIJ,QAAQrC,GAAG,CAAC,SAACoB;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL;yBAAO;;oBAChElC,SAAS;oBAEb,4FAA4F;oBAC5F,mFAAmF;oBAC7EqD,cAAcD,OAAOI,GAAG,CAAC;yBAC3BH,aAAAA;;;;oBACS;;wBAAMpB,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAMiD;;;oBAA3ErD,SAAAA,SAAU,AAAC,cAA8EA,MAAM;yBAG3F,CAACqD,YAAYV,QAAQ,EAArB;;;;oBAAuB;;wBAAME,SAASlD,MAAMC;;;oBAArB;;;oBAG7B,wFAAwF;oBACxF,yFAAyF;oBACzF,8EAA8E;oBACxE0D,aAAaF,OAAOI,GAAG,CAAC;yBAC1BF,YAAAA;;;;yBACEA,WAAWX,QAAQ,EAAnBW;;;;oBAAqB;;wBAAMN,UAAUrD,MAAMC;;;oBAAtB;;;;;;oBACpB;;wBAAMiD,SAASlD,MAAMC;;;oBAArB;;;oBAGF,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAgBuD;;;2BAAhB,6BAAA,QAAA;;;;oBAAMjB,SAAN;oBACH,IAAIA,OAAOK,IAAI,KAAK,WAAWL,OAAOK,IAAI,KAAK,QAAQ;;;;oBAC5C;;wBAAMN,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAM8B;;;oBAA3ElC,SAAAA,SAAU,AAAC,cAAyEA,MAAM;;;oBAFvF;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAIL;;wBAAO;4BAAEA,QAAAA;wBAAO;;;;IAClB;;AAIO,SAASP,cAAcE,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB;IACrG,IAAMQ,OAAO,IAAIqD,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA,MAAQC;mBAAgBC,IAAAA,sBAAS,EAACjE,MAAMO,KAAKC,SAASP,SAASQ,MAAMuD;;QACrEE,YAAY,SAAZA,WAAaC;mBAAUA,SAAS,UAAUpE,iBAAiBC,MAAMC,WAAWK,kBAAkBN,MAAMO,KAAKC,SAASP,SAASQ;;QAC3H2D,oBAAoB,SAApBA,mBAAqBZ;mBAAYD,yBAAyBvD,MAAMO,KAAKC,SAASP,SAASQ,MAAM+C;;QAC7Fa,OAAO,SAAPA;mBAAa5D,KAAK4D,KAAK;;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport type { Stages } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","nullEmbedVectors","conn","dialect","withTransaction","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","existingPaths","files","docs","paths","delta","prepare","Set","all","map","r","path","listFiles","filter","f","embed","has","relPath","length","reparseFiles","undefined","d","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","feature","docsForFeature","NARROW_FEATURE_TABLE","name","FEATURES","find","Error","turnedOn","afterReconcile","nullRank","columns","getColumns","rerunRank","rank","invalidateFeatureToggles","toggles","byName","linksToggle","rankToggle","Map","get","ParsePool","build","forcedPaths","reconcile","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":";;;;+BA8JgBA;;;eAAAA;;;uBA7JM;uBACG;sBACJ;wBAEK;sBACA;yBACG;8BAGQ;2BACX;wBACC;6BAEK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBhC,6FAA6F;AAC7F,wEAAwE;AACxE,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;;;;;oBACzE;;wBAAMC,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAAsDF,QAAQG,SAAS;;;oBAAnH;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;;IACrB;;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,SAAeC,kBAAkBN,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe;;YACnHC,cACAC,kBAGAC,OAGEC,MACFC,OACAC;;;;oBATe;;wBAAMf,KAAKgB,OAAO,CAAC;;;oBAAlCN,eAAe;wBACKO;oBAAK;;wBAAMP,aAAaQ,GAAG;;;oBAA/CP,gBAAgB,IAAA,CAAA,EAAA,MAAIM;;wBAAK,cAA0BE,GAAG,CAAC,SAACC;mCAAM,AAACA,EAAuBC,IAAI;;sBAAC;oBACjG,8FAA8F;oBAC9F,4FAA4F;oBACtFT,QAAQU,IAAAA,mBAAS,EAACf,KAAKC,SAASe,MAAM,CAAC,SAACC;+BAAMA,EAAEC,KAAK,IAAId,cAAce,GAAG,CAACF,EAAEG,OAAO;;oBAC1F,IAAIf,MAAMgB,MAAM,KAAK,GAAG;;wBAAO;4BAAEvB,QAAQ;wBAAE;;oBAE1B;;wBAAMwB,IAAAA,uBAAY,EAACjB;4BAAQa,cAAK;2BAAGlB,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAA9EI,OAAS,cAATA;oBACFC,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACjCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,KAAK;wBAAMC,QAAQ;oBAAK;oBAChF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQyB,eACAA;;;;4CADN;;iDAAMA,gBAAAA,cAAK,CAACU,MAAM,cAAZV,oCAAAA,mBAAAA,cAAK,EAAUzB,MAAMc,OAAOC;;;4CAAlC;4CACA;;iDAAMU,eAAAA,cAAK,CAACW,KAAK,cAAXX,mCAAAA,kBAAAA,cAAK,EACTzB,MACAa,KAAKM,GAAG,CAAC,SAACY;2DAAO;wDAAEV,MAAMU,EAAEJ,OAAO;wDAAEU,WAAWN,EAAEM,SAAS,CAACZ,KAAK;oDAAC;oDACjEV;;;4CAHF;;;;;;4BAKF;2BACAd,QAAQG,SAAS;;;oBAVnB;oBAYA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,SAAeU,uBAAuBtC,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE8B,MAAqB;;YAC/IC,OACAC,SAQA7B,OACEC,MACF6B,gBACA5B,OAIAC;;;;oBAhBAyB,QAAQG,oCAAoB,CAACJ,OAAOK,IAAI,CAAC;oBACzCH,UAAUI,iBAAQ,CAACC,IAAI,CAAC,SAACtB;+BAAMA,EAAEoB,IAAI,KAAKL,OAAOK,IAAI;;oBAC3D,IAAI,CAACH,SAAS,MAAM,IAAIM,MAAM,AAAC,wDAAmE,OAAZR,OAAOK,IAAI,EAAC;yBAE9F,CAACL,OAAOS,QAAQ,EAAhB;;;;oBACF;;wBAAM9C,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;2BAAUvC,QAAQG,SAAS;;;oBAAtF;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;oBAGfO,QAAQU,IAAAA,mBAAS,EAACf,KAAKC;oBACZ;;wBAAMqB,IAAAA,uBAAY,EAACjB;4BAAQ6B;2BAAUlC,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAAhFI,OAAS,cAATA;oBACF6B,iBAAiB7B,KAAKM,GAAG,CAAC,SAACY;+BAAO;4BAAEV,MAAMU,EAAEJ,OAAO;4BAAEU,WAAWN,EAAEM,SAAS,CAACI,QAAQG,IAAI,CAAC;wBAAC;;oBAC1F9B,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACvC,gGAAgG;oBAChG,8FAA8F;oBAC9F,8CAA8C;oBACxCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,OAAOnB;wBAAOoB,QAAQ;oBAAK;oBACnF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCAEQyC,gBACAA;;;;4CAFN;;gDAAMzC,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;;;4CAA/B;4CACA;;iDAAMC,iBAAAA,QAAQL,KAAK,cAAbK,qCAAAA,oBAAAA,SAAgBzC,MAAM0C,gBAAgB3B;;;4CAA5C;4CACA;;iDAAM0B,0BAAAA,QAAQQ,cAAc,cAAtBR,8CAAAA,6BAAAA,SAAyBzC,MAAMe;;;4CAArC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBAPnB;oBASA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,SAAesB,SAASlD,IAAgB,EAAEC,OAAyB;;YAC3DkD;;;;oBAAU;;wBAAMC,IAAAA,oBAAU,EAACpD;;;oBAA3BmD,UAAU;oBAChB,IAAI,CAACA,QAAQzB,GAAG,CAAC,UAAU;;;oBAC3B;;wBAAMxB,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAA0CF,QAAQG,SAAS;;;oBAAvG;;;;;;IACF;;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAAeiD,UAAUrD,IAAgB,EAAEC,OAAyB;;YAC5Dc;;;;oBAAAA,QAAwB;wBAAEH,KAAK;wBAAMoB,QAAQ;wBAAMC,KAAK;wBAAMC,QAAQ;oBAAK;oBACjF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQsD;;;;4CAAN;;iDAAMA,uBAAAA,YAAI,CAACL,cAAc,cAAnBK,2CAAAA,0BAAAA,YAAI,EAAkBtD,MAAMe;;;4CAAlC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBALnB;;;;;;IAOF;;AAEA,SAAemD,yBAAyBvD,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE+C,OAAwB;;YACpJC,QACFpD,QAIEqD,aAWAC,YAMD,2BAAA,mBAAA,gBAAA,WAAA,OAAMpB;;;;oBAtBLkB,SAAS,IAAIG,IAAIJ,QAAQrC,GAAG,CAAC,SAACoB;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL;yBAAO;;oBAChElC,SAAS;oBAEb,4FAA4F;oBAC5F,mFAAmF;oBAC7EqD,cAAcD,OAAOI,GAAG,CAAC;yBAC3BH,aAAAA;;;;oBACS;;wBAAMpB,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAMiD;;;oBAA3ErD,SAAAA,SAAU,AAAC,cAA8EA,MAAM;yBAG3F,CAACqD,YAAYV,QAAQ,EAArB;;;;oBAAuB;;wBAAME,SAASlD,MAAMC;;;oBAArB;;;oBAG7B,wFAAwF;oBACxF,yFAAyF;oBACzF,8EAA8E;oBACxE0D,aAAaF,OAAOI,GAAG,CAAC;yBAC1BF,YAAAA;;;;yBACEA,WAAWX,QAAQ,EAAnBW;;;;oBAAqB;;wBAAMN,UAAUrD,MAAMC;;;oBAAtB;;;;;;oBACpB;;wBAAMiD,SAASlD,MAAMC;;;oBAArB;;;oBAGF,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAgBuD;;;2BAAhB,6BAAA,QAAA;;;;oBAAMjB,SAAN;oBACH,IAAIA,OAAOK,IAAI,KAAK,WAAWL,OAAOK,IAAI,KAAK,QAAQ;;;;oBAC5C;;wBAAMN,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAM8B;;;oBAA3ElC,SAAAA,SAAU,AAAC,cAAyEA,MAAM;;;oBAFvF;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAIL;;wBAAO;4BAAEA,QAAAA;wBAAO;;;;IAClB;;AAIO,SAASP,cAAcE,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB;IACrG,IAAMQ,OAAO,IAAIqD,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA,MAAQC;mBAAgBC,IAAAA,sBAAS,EAACjE,MAAMO,KAAKC,SAASP,SAASQ,MAAMuD;;QACrEE,YAAY,SAAZA,WAAaC;mBAAUA,SAAS,UAAUpE,iBAAiBC,MAAMC,WAAWK,kBAAkBN,MAAMO,KAAKC,SAASP,SAASQ;;QAC3H2D,oBAAoB,SAApBA,mBAAqBZ;mBAAYD,yBAAyBvD,MAAMO,KAAKC,SAASP,SAASQ,MAAM+C;;QAC7Fa,OAAO,SAAPA;mBAAa5D,KAAK4D,KAAK;;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}