sensemaking 0.24.7 → 0.25.0

package/dist/esm/scan/index.js

@@ -5,9 +5,7 @@ import { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_
 // Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.
 export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
 export { listFiles, toPosixPath } from './list.js';
-export function parseFile(file, extractors = [], cfg) {
-    // Preserve the authored bytes: feature extractors and section offsets use the raw document.
-    const raw = readFileSync(file.absPath, 'utf8');
+export function parseSource(file, raw, extractors = [], cfg) {
     const warnings = [];
     const { fm, body: content } = splitFrontmatter(raw);
     const { data, parseError } = fm === null ? {
@@ -45,6 +43,7 @@ export function parseFile(file, extractors = [], cfg) {
             presets: file.presets,
             data: mapped,
             parseError,
+            source: raw,
             search,
             extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
                 var _f_extract;
@@ -57,3 +56,8 @@ export function parseFile(file, extractors = [], cfg) {
         warnings
     };
 }
+export function parseFile(file, extractors = [], cfg) {
+    // Preserve the authored bytes: feature extractors, indexed snippets, and section offsets use
+    // the same decoded string from this one read.
+    return parseSource(file, readFileSync(file.absPath, 'utf8'), extractors, cfg);
+}

package/dist/esm/scan/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors and section offsets use the raw document.\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,4FAA4F;IAC5F,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // Exact decoded source bytes read for this parse generation. Stored outside the FTS table so\n  // snippets and delayed embedding never consult a newer filesystem generation.\n  source: string;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseSource(file: FileStat, raw: string, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      source: raw,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors, indexed snippets, and section offsets use\n  // the same decoded string from this one read.\n  return parseSource(file, readFileSync(file.absPath, 'utf8'), extractors, cfg);\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseSource","file","raw","extractors","cfg","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","source","extracted","fromEntries","filter","f","extract","map","name","parseFile","absPath"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAqBnD,OAAO,SAASC,YAAYC,IAAc,EAAEC,GAAW,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAC/F,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGZ,iBAAiBM;IAC/C,MAAM,EAAEO,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIhB,iBAAiBO,KAAKU,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAId,iBAAiBqB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEhB,KAAKU,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ1B,SAASiB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY3B,kBAAkB2B,UAAUC,OAAOC,KAAK,CAACC,KAAKhC,KAAK,CAAC6B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEhB,KAAKU,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASjC,MAAMmB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAO/B,cAAcgB,KAAKe,KAAK;QAAGC,SAAShC,cAAcgB,KAAKgB,OAAO;QAAGC,MAAMpC,eAAegC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASV,KAAKU,OAAO;YACrBiB,SAAS3B,KAAK2B,OAAO;YACrBC,SAAS5B,KAAK4B,OAAO;YACrBC,MAAM7B,KAAK6B,IAAI;YACfC,SAAS9B,KAAK8B,OAAO;YACrBtB,MAAMG;YACNF;YACAsB,QAAQ9B;YACRqB;YACAU,WAAWnB,OAAOoB,WAAW,CAAC/B,WAAWgC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYlC,KAAKM,SAASe,QAAQd,MAAML,KAAKkB;iBAAQ;;QAC7I;QACAjB;IACF;AACF;AAEA,OAAO,SAASmC,UAAUvC,IAAc,EAAEE,aAAwB,EAAE,EAAEC,GAAY;IAChF,6FAA6F;IAC7F,8CAA8C;IAC9C,OAAOJ,YAAYC,MAAMb,aAAaa,KAAKwC,OAAO,EAAE,SAAStC,YAAYC;AAC3E"}

package/dist/esm/scan/pool.d.ts

@@ -9,8 +9,13 @@ export type FileResult = {
 };
 export declare class ParsePool {
     private pool;
+    private workerData;
+    private operation;
     poolsCreated: number;
+    private enqueue;
     private ensure;
     run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]>;
+    private dispatch;
+    private destroy;
     close(): Promise<void>;
 }

package/dist/esm/scan/pool.js

@@ -1,12 +1,19 @@
 import { createRequire } from 'node:module';
+import { isDeepStrictEqual } from 'node:util';
 import { resolveWorkerFile } from '../lib/worker-file.js';
 import { reviveError } from './worker-error.js';
 // Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
 const _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;
-// One Tinypool per instance, created lazily on first dispatch and reused by every later call:
-// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.
+// Created lazily and reused while dispatch context stays equivalent. A builder owns one instance
+// for its lifetime instead of paying pool startup for unchanged reconciles.
 export class ParsePool {
-    ensure(workerData, maxWorkers) {
+    enqueue(operation) {
+        const result = this.operation.then(operation);
+        this.operation = result.then(()=>undefined, ()=>undefined);
+        return result;
+    }
+    async ensure(workerData, maxWorkers) {
+        if (this.pool && !isDeepStrictEqual(this.workerData, workerData)) await this.destroy();
         if (!this.pool) {
             const { Tinypool } = _require('tinypool');
             this.pool = new Tinypool({
@@ -15,24 +22,27 @@ export class ParsePool {
                 maxThreads: maxWorkers,
                 workerData
             });
+            this.workerData = workerData;
             this.poolsCreated++;
         }
         return this.pool;
     }
     // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
     // extracted text and feature values, never the token tree.
-    async run(files, features, cfg, onParsed, maxWorkers) {
-        // cfg and the feature selection are constant for the dispatch, so they cross once per worker
-        // as workerData. Features carry closures and cannot cross at all; only their names do.
+    run(files, features, cfg, onParsed, maxWorkers) {
+        // Snapshot at invocation so caller mutation cannot change queued work or its reuse baseline.
         const workerData = {
-            cfg,
-            featureNames: features.map((feature)=>feature.name)
+            cfg: structuredClone(cfg),
+            featureNames: [
+                ...new Set(features.map((feature)=>feature.name))
+            ].sort()
         };
-        const pool = this.ensure(workerData, maxWorkers);
+        return this.enqueue(()=>this.dispatch(files, workerData, onParsed, maxWorkers));
+    }
+    async dispatch(files, workerData, onParsed, maxWorkers) {
+        const pool = await this.ensure(workerData, maxWorkers);
         let done = 0;
-        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
-        // regardless of task completion order, which first-seen column order depends on downstream.
-        return Promise.all(files.map(async (file)=>{
+        const settled = await Promise.allSettled(files.map(async (file)=>{
             const result = await pool.run(file);
             if (!result.ok) throw reviveError(result.error);
             onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
@@ -42,17 +52,32 @@ export class ParsePool {
                 parseMs: result.parseMs
             };
         }));
+        const results = [];
+        const failures = [];
+        for (const result of settled){
+            if (result.status === 'fulfilled') results.push(result.value);
+            else failures.push(result.reason);
+        }
+        if (failures.length === 1) throw failures[0];
+        if (failures.length > 1) throw new AggregateError(failures, `${failures.length} parse tasks failed`);
+        // allSettled preserves input order, which downstream first-seen column order depends on.
+        return results;
     }
-    // A pool never created costs nothing to destroy.
-    async close() {
+    async destroy() {
         if (!this.pool) return;
         const pool = this.pool;
         this.pool = undefined;
+        this.workerData = undefined;
         await pool.destroy();
     }
+    // A pool never created costs nothing to destroy. Queuing lets active dispatches drain first.
+    close() {
+        return this.enqueue(()=>this.destroy());
+    }
     constructor(){
-        // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;
-        // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.
+        this.operation = Promise.resolve();
+        // Tinypools this instance has constructed. Repeated equivalent dispatches leave this unchanged;
+        // a context change or close followed by run constructs another pool.
         this.poolsCreated = 0;
     }
 }

package/dist/esm/scan/pool.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { resolveWorkerFile } from '../lib/worker-file.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile('parse'), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the token tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["createRequire","resolveWorkerFile","reviveError","_require","require","url","ParsePool","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","featureNames","map","feature","name","done","Promise","all","file","result","ok","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAM5C,SAASC,iBAAiB,QAAQ,wBAAwB;AAI1D,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcJ,cAAc,YAAYK,GAAG,IAAID;AAKnF,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAME;IAMHC,OAAOC,UAA2B,EAAEC,UAAkB,EAAuB;QACnF,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,MAAM,EAAEC,QAAQ,EAAE,GAAGR,SAAS;YAC9B,IAAI,CAACO,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUX,kBAAkB;gBAAUY,YAAYJ;gBAAYK,YAAYL;gBAAYD;YAAW;YAC5H,IAAI,CAACO,YAAY;QACnB;QACA,OAAO,IAAI,CAACL,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,MAAMM,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEX,UAAkB,EAAyB;QACxJ,6FAA6F;QAC7F,uFAAuF;QACvF,MAAMD,aAA8B;YAAEW;YAAKE,cAAcH,SAASI,GAAG,CAAC,CAACC,UAAYA,QAAQC,IAAI;QAAE;QACjG,MAAMd,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;QACrC,IAAIgB,OAAO;QACX,0FAA0F;QAC1F,4FAA4F;QAC5F,OAAOC,QAAQC,GAAG,CAChBV,MAAMK,GAAG,CAAC,OAAOM;YACf,MAAMC,SAAU,MAAMnB,KAAKM,GAAG,CAACY;YAC/B,IAAI,CAACC,OAAOC,EAAE,EAAE,MAAM5B,YAAY2B,OAAOE,KAAK;YAC9CX,qBAAAA,+BAAAA,SAAW,EAAEK;YACb,OAAO;gBAAEO,KAAKH,OAAOG,GAAG;gBAAEC,UAAUJ,OAAOI,QAAQ;gBAAEC,SAASL,OAAOK,OAAO;YAAC;QAC/E;IAEJ;IAEA,iDAAiD;IACjD,MAAMC,QAAuB;QAC3B,IAAI,CAAC,IAAI,CAACzB,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAG0B;QACZ,MAAM1B,KAAK2B,OAAO;IACpB;;QAvCA,2FAA2F;QAC3F,uGAAuG;aACvGtB,eAAe;;AAsCjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { isDeepStrictEqual } from 'node:util';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { resolveWorkerFile } from '../lib/worker-file.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// Created lazily and reused while dispatch context stays equivalent. A builder owns one instance\n// for its lifetime instead of paying pool startup for unchanged reconciles.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  private workerData: ParseWorkerData | undefined;\n  private operation: Promise<void> = Promise.resolve();\n  // Tinypools this instance has constructed. Repeated equivalent dispatches leave this unchanged;\n  // a context change or close followed by run constructs another pool.\n  poolsCreated = 0;\n\n  private enqueue<T>(operation: () => Promise<T>): Promise<T> {\n    const result = this.operation.then(operation);\n    this.operation = result.then(\n      () => undefined,\n      () => undefined\n    );\n    return result;\n  }\n\n  private async ensure(workerData: ParseWorkerData, maxWorkers: number): Promise<TinypoolNS.Tinypool> {\n    if (this.pool && !isDeepStrictEqual(this.workerData, workerData)) await this.destroy();\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile('parse'), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.workerData = workerData;\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the token tree.\n  run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // Snapshot at invocation so caller mutation cannot change queued work or its reuse baseline.\n    const workerData: ParseWorkerData = { cfg: structuredClone(cfg), featureNames: [...new Set(features.map((feature) => feature.name))].sort() };\n    return this.enqueue(() => this.dispatch(files, workerData, onParsed, maxWorkers));\n  }\n\n  private async dispatch(files: FileStat[], workerData: ParseWorkerData, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    const pool = await this.ensure(workerData, maxWorkers);\n    let done = 0;\n    const settled = await Promise.allSettled(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n    const results: FileResult[] = [];\n    const failures: unknown[] = [];\n    for (const result of settled) {\n      if (result.status === 'fulfilled') results.push(result.value);\n      else failures.push(result.reason);\n    }\n    if (failures.length === 1) throw failures[0];\n    if (failures.length > 1) throw new AggregateError(failures, `${failures.length} parse tasks failed`);\n    // allSettled preserves input order, which downstream first-seen column order depends on.\n    return results;\n  }\n\n  private async destroy(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    this.workerData = undefined;\n    await pool.destroy();\n  }\n\n  // A pool never created costs nothing to destroy. Queuing lets active dispatches drain first.\n  close(): Promise<void> {\n    return this.enqueue(() => this.destroy());\n  }\n}\n"],"names":["createRequire","isDeepStrictEqual","resolveWorkerFile","reviveError","_require","require","url","ParsePool","enqueue","operation","result","then","undefined","ensure","workerData","maxWorkers","pool","destroy","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","structuredClone","featureNames","Set","map","feature","name","sort","dispatch","done","settled","Promise","allSettled","file","ok","error","doc","warnings","parseMs","results","failures","status","push","value","reason","length","AggregateError","close","resolve"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,iBAAiB,QAAQ,YAAY;AAM9C,SAASC,iBAAiB,QAAQ,wBAAwB;AAI1D,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcL,cAAc,YAAYM,GAAG,IAAID;AAKnF,iGAAiG;AACjG,4EAA4E;AAC5E,OAAO,MAAME;IAQHC,QAAWC,SAA2B,EAAc;QAC1D,MAAMC,SAAS,IAAI,CAACD,SAAS,CAACE,IAAI,CAACF;QACnC,IAAI,CAACA,SAAS,GAAGC,OAAOC,IAAI,CAC1B,IAAMC,WACN,IAAMA;QAER,OAAOF;IACT;IAEA,MAAcG,OAAOC,UAA2B,EAAEC,UAAkB,EAAgC;QAClG,IAAI,IAAI,CAACC,IAAI,IAAI,CAACf,kBAAkB,IAAI,CAACa,UAAU,EAAEA,aAAa,MAAM,IAAI,CAACG,OAAO;QACpF,IAAI,CAAC,IAAI,CAACD,IAAI,EAAE;YACd,MAAM,EAAEE,QAAQ,EAAE,GAAGd,SAAS;YAC9B,IAAI,CAACY,IAAI,GAAG,IAAIE,SAAS;gBAAEC,UAAUjB,kBAAkB;gBAAUkB,YAAYL;gBAAYM,YAAYN;gBAAYD;YAAW;YAC5H,IAAI,CAACA,UAAU,GAAGA;YAClB,IAAI,CAACQ,YAAY;QACnB;QACA,OAAO,IAAI,CAACN,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3DO,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEZ,UAAkB,EAAyB;QAClJ,6FAA6F;QAC7F,MAAMD,aAA8B;YAAEY,KAAKE,gBAAgBF;YAAMG,cAAc;mBAAI,IAAIC,IAAIL,SAASM,GAAG,CAAC,CAACC,UAAYA,QAAQC,IAAI;aAAG,CAACC,IAAI;QAAG;QAC5I,OAAO,IAAI,CAAC1B,OAAO,CAAC,IAAM,IAAI,CAAC2B,QAAQ,CAACX,OAAOV,YAAYa,UAAUZ;IACvE;IAEA,MAAcoB,SAASX,KAAiB,EAAEV,UAA2B,EAAEa,QAA8C,EAAEZ,UAAkB,EAAyB;QAChK,MAAMC,OAAO,MAAM,IAAI,CAACH,MAAM,CAACC,YAAYC;QAC3C,IAAIqB,OAAO;QACX,MAAMC,UAAU,MAAMC,QAAQC,UAAU,CACtCf,MAAMO,GAAG,CAAC,OAAOS;YACf,MAAM9B,SAAU,MAAMM,KAAKO,GAAG,CAACiB;YAC/B,IAAI,CAAC9B,OAAO+B,EAAE,EAAE,MAAMtC,YAAYO,OAAOgC,KAAK;YAC9Cf,qBAAAA,+BAAAA,SAAW,EAAES;YACb,OAAO;gBAAEO,KAAKjC,OAAOiC,GAAG;gBAAEC,UAAUlC,OAAOkC,QAAQ;gBAAEC,SAASnC,OAAOmC,OAAO;YAAC;QAC/E;QAEF,MAAMC,UAAwB,EAAE;QAChC,MAAMC,WAAsB,EAAE;QAC9B,KAAK,MAAMrC,UAAU2B,QAAS;YAC5B,IAAI3B,OAAOsC,MAAM,KAAK,aAAaF,QAAQG,IAAI,CAACvC,OAAOwC,KAAK;iBACvDH,SAASE,IAAI,CAACvC,OAAOyC,MAAM;QAClC;QACA,IAAIJ,SAASK,MAAM,KAAK,GAAG,MAAML,QAAQ,CAAC,EAAE;QAC5C,IAAIA,SAASK,MAAM,GAAG,GAAG,MAAM,IAAIC,eAAeN,UAAU,GAAGA,SAASK,MAAM,CAAC,mBAAmB,CAAC;QACnG,yFAAyF;QACzF,OAAON;IACT;IAEA,MAAc7B,UAAyB;QACrC,IAAI,CAAC,IAAI,CAACD,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAGJ;QACZ,IAAI,CAACE,UAAU,GAAGF;QAClB,MAAMI,KAAKC,OAAO;IACpB;IAEA,6FAA6F;IAC7FqC,QAAuB;QACrB,OAAO,IAAI,CAAC9C,OAAO,CAAC,IAAM,IAAI,CAACS,OAAO;IACxC;;aAnEQR,YAA2B6B,QAAQiB,OAAO;QAClD,gGAAgG;QAChG,qEAAqE;aACrEjC,eAAe;;AAiEjB"}

package/dist/esm/store/builder.d.ts

@@ -1,4 +1,4 @@
-import type { Config } from '../config/index.js';
+import { type Config } from '../config/index.js';
 import type { EmbedChangeKind } from './embed-scope.js';
 import type { FeatureToggle } from './feature-scope.js';
 import { type ReconcilePaths } from './reconcile.js';
@@ -10,6 +10,11 @@ export interface Builder {
         warnings: string[];
         stages: Stages;
     }>;
+    recover(): Promise<{
+        parsed: number;
+        warnings: string[];
+        stages: Stages;
+    }>;
     invalidate(kind: EmbedChangeKind): Promise<{
         parsed: number;
     }>;

package/dist/esm/store/builder.js

@@ -1,3 +1,4 @@
+import { featureEnabled } from '../config/index.js';
 import { embed } from '../features/embed.js';
 import { FEATURES } from '../features/index.js';
 import { rank } from '../features/rank.js';
@@ -146,12 +147,30 @@ async function invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggl
         parsed
     };
 }
+async function recoverIncomplete(conn, cfg, paths, dialect, pool) {
+    const disabled = FEATURES.filter((feature)=>!featureEnabled(cfg, feature.name));
+    for (const feature of disabled){
+        if (feature.name !== 'embed') await feature.schema(conn);
+    }
+    const toggles = disabled.filter((feature)=>feature.name !== 'embed').map((feature)=>({
+            name: feature.name,
+            turnedOn: false
+        }));
+    await invalidateFeatureToggles(conn, cfg, paths.rootDir, dialect, pool, toggles);
+    if (disabled.some((feature)=>feature.name === 'embed')) {
+        const embeddingsTable = await conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'embeddings'");
+        if ((await embeddingsTable.all()).length > 0) await withTransaction(conn, ()=>conn.exec('DELETE FROM embeddings'), dialect.beginMode());
+    }
+    const forcedPaths = new Set(listFiles(cfg, paths.rootDir).map((file)=>file.relPath));
+    return reconcile(conn, cfg, paths, dialect, pool, forcedPaths);
+}
 // The pool is created at most once, lazily, on whichever build() or invalidate() call first
 // needs it, and reused by every later call on this instance.
 export function createBuilder(conn, cfg, paths, dialect) {
     const pool = new ParsePool();
     return {
         build: (forcedPaths)=>reconcile(conn, cfg, paths, dialect, pool, forcedPaths),
+        recover: ()=>recoverIncomplete(conn, cfg, paths, dialect, pool),
         invalidate: (kind)=>kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, paths.rootDir, dialect, pool),
         invalidateFeatures: (toggles)=>invalidateFeatureToggles(conn, cfg, paths.rootDir, dialect, pool, toggles),
         close: ()=>pool.close(),

package/dist/esm/store/builder.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { type ReconcilePaths, reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport type { Stages } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: nothing stale to diff against. Links resolves\n  // dst inside store() itself when added.length equals files.length (a cold build).\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, paths, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, paths.rootDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, paths.rootDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["embed","FEATURES","rank","listFiles","ParsePool","reparseFiles","NARROW_FEATURE_TABLE","reconcile","getColumns","withTransaction","nullEmbedVectors","conn","dialect","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","prepare","existingPaths","Set","all","map","r","path","files","filter","f","has","relPath","length","docs","undefined","paths","d","delta","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","name","feature","find","Error","turnedOn","docsForFeature","afterReconcile","nullRank","columns","rerunRank","invalidateFeatureToggles","toggles","byName","Map","linksToggle","get","rankToggle","createBuilder","build","forcedPaths","invalidate","kind","rootDir","invalidateFeatures","close","poolsCreated"],"mappings":"AACA,SAASA,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,IAAI,QAAQ,sBAAsB;AAE3C,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,YAAY,QAAQ,qBAAqB;AAGlD,SAASC,oBAAoB,QAAQ,qBAAqB;AAC1D,SAA8BC,SAAS,QAAQ,iBAAiB;AAChE,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,eAAe,QAAQ,mBAAmB;AAsBnD,6FAA6F;AAC7F,wEAAwE;AACxE,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;IACzE,MAAMH,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,sDAAsDD,QAAQE,SAAS;IACnH,OAAO;QAAEC,QAAQ;IAAE;AACrB;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,eAAeC,kBAAkBL,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe;IACzH,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,gBAAgB,IAAIC,IAAI,AAAC,CAAA,MAAMH,aAAaI,GAAG,EAAC,EAAGC,GAAG,CAAC,CAACC,IAAM,AAACA,EAAuBC,IAAI;IAChG,8FAA8F;IAC9F,4FAA4F;IAC5F,MAAMC,QAAQzB,UAAUc,KAAKC,SAASW,MAAM,CAAC,CAACC,IAAMA,EAAE9B,KAAK,IAAIsB,cAAcS,GAAG,CAACD,EAAEE,OAAO;IAC1F,IAAIJ,MAAMK,MAAM,KAAK,GAAG,OAAO;QAAElB,QAAQ;IAAE;IAE3C,MAAM,EAAEmB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAAC5B;KAAM,EAAEiB,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACtF,MAAMiB,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IAChF,MAAMhC,gBACJE,MACA;YACQX,eACAA;QADN,QAAMA,gBAAAA,MAAM0C,MAAM,cAAZ1C,oCAAAA,mBAAAA,OAAeW,MAAMyB,OAAOE;QAClC,QAAMtC,eAAAA,MAAM2C,KAAK,cAAX3C,mCAAAA,kBAAAA,OACJW,MACAuB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;gBAAEV,MAAMU,EAAEL,OAAO;gBAAEY,WAAWP,EAAEO,SAAS,CAAC5C,KAAK;YAAC,CAAA,IACjEsC;IAEJ,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,eAAeY,uBAAuBlC,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAE2B,MAAqB;IACrJ,MAAMC,QAAQzC,oBAAoB,CAACwC,OAAOE,IAAI,CAAC;IAC/C,MAAMC,UAAUhD,SAASiD,IAAI,CAAC,CAACpB,IAAMA,EAAEkB,IAAI,KAAKF,OAAOE,IAAI;IAC3D,IAAI,CAACC,SAAS,MAAM,IAAIE,MAAM,CAAC,qDAAqD,EAAEL,OAAOE,IAAI,CAAC,CAAC,CAAC;IAEpG,IAAI,CAACF,OAAOM,QAAQ,EAAE;QACpB,MAAM3C,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO,GAAGnC,QAAQE,SAAS;QACtF,OAAO;YAAEC,QAAQ;QAAE;IACrB;IAEA,MAAMa,QAAQzB,UAAUc,KAAKC;IAC7B,MAAM,EAAEgB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAACqB;KAAQ,EAAEhC,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACxF,MAAMkC,iBAAiBnB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;YAAEV,MAAMU,EAAEL,OAAO;YAAEY,WAAWP,EAAEO,SAAS,CAACK,QAAQD,IAAI,CAAC;QAAC,CAAA;IAChG,MAAMZ,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,8FAA8F;IAC9F,kFAAkF;IAClF,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAOJ;QAAOK,UAAU,EAAE;IAAC;IACnF,MAAMhC,gBACJE,MACA;YAEQsC,gBACAA;QAFN,MAAMtC,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO;QACtC,QAAME,iBAAAA,QAAQN,KAAK,cAAbM,qCAAAA,oBAAAA,SAAgBtC,MAAM0C,gBAAgBf;QAC5C,QAAMW,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyBtC,MAAM2B;IACvC,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,eAAesB,SAAS5C,IAAgB,EAAEC,OAAyB;IACjE,MAAM4C,UAAU,MAAMhD,WAAWG;IACjC,IAAI,CAAC6C,QAAQzB,GAAG,CAAC,UAAU;IAC3B,MAAMtB,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,0CAA0CD,QAAQE,SAAS;AACzG;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,eAAe2C,UAAU9C,IAAgB,EAAEC,OAAyB;IAClE,MAAM0B,QAAwB;QAAEV,OAAO,EAAE;QAAEW,UAAU,EAAE;QAAEC,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IACjF,MAAMhC,gBACJE,MACA;YACQT;QAAN,QAAMA,uBAAAA,KAAKoD,cAAc,cAAnBpD,2CAAAA,0BAAAA,MAAsBS,MAAM2B;IACpC,GACA1B,QAAQE,SAAS;AAErB;AAEA,eAAe4C,yBAAyB/C,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAEwC,OAAwB;IAC1J,MAAMC,SAAS,IAAIC,IAAIF,QAAQlC,GAAG,CAAC,CAACqB,SAAW;YAACA,OAAOE,IAAI;YAAEF;SAAO;IACpE,IAAI/B,SAAS;IAEb,4FAA4F;IAC5F,mFAAmF;IACnF,MAAM+C,cAAcF,OAAOG,GAAG,CAAC;IAC/B,IAAID,aAAa;QACf/C,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2C,YAAW,EAAG/C,MAAM;QAC/F,4FAA4F;QAC5F,0FAA0F;QAC1F,IAAI,CAAC+C,YAAYV,QAAQ,EAAE,MAAMG,SAAS5C,MAAMC;IAClD;IAEA,wFAAwF;IACxF,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMoD,aAAaJ,OAAOG,GAAG,CAAC;IAC9B,IAAIC,YAAY;QACd,IAAIA,WAAWZ,QAAQ,EAAE,MAAMK,UAAU9C,MAAMC;aAC1C,MAAM2C,SAAS5C,MAAMC;IAC5B;IAEA,KAAK,MAAMkC,UAAUa,QAAS;QAC5B,IAAIb,OAAOE,IAAI,KAAK,WAAWF,OAAOE,IAAI,KAAK,QAAQ;QACvDjC,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2B,OAAM,EAAG/B,MAAM;IAC5F;IACA,OAAO;QAAEA;IAAO;AAClB;AAEA,4FAA4F;AAC5F,6DAA6D;AAC7D,OAAO,SAASkD,cAActD,IAAgB,EAAEM,GAAW,EAAEmB,KAAqB,EAAExB,OAAyB;IAC3G,MAAMO,OAAO,IAAIf;IACjB,OAAO;QACL8D,OAAO,CAACC,cAAgB5D,UAAUI,MAAMM,KAAKmB,OAAOxB,SAASO,MAAMgD;QACnEC,YAAY,CAACC,OAAUA,SAAS,UAAU3D,iBAAiBC,MAAMC,WAAWI,kBAAkBL,MAAMM,KAAKmB,MAAMkC,OAAO,EAAE1D,SAASO;QACjIoD,oBAAoB,CAACZ,UAAYD,yBAAyB/C,MAAMM,KAAKmB,MAAMkC,OAAO,EAAE1D,SAASO,MAAMwC;QACnGa,OAAO,IAAMrD,KAAKqD,KAAK;QACvB,IAAIC;YACF,OAAOtD,KAAKsD,YAAY;QAC1B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import { type Config, featureEnabled } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { type ReconcilePaths, reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport type { Stages } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }>;\n  // Repairs an incomplete generation in place, including state owned by disabled features.\n  recover(): Promise<{ parsed: number; warnings: string[]; stages: Stages }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: nothing stale to diff against. Links resolves\n  // dst inside store() itself when added.length equals files.length (a cold build).\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\nasync function recoverIncomplete(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const disabled = FEATURES.filter((feature) => !featureEnabled(cfg, feature.name));\n  for (const feature of disabled) {\n    if (feature.name !== 'embed') await feature.schema(conn);\n  }\n\n  const toggles: FeatureToggle[] = disabled.filter((feature) => feature.name !== 'embed').map((feature) => ({ name: feature.name, turnedOn: false }));\n  await invalidateFeatureToggles(conn, cfg, paths.rootDir, dialect, pool, toggles);\n  if (disabled.some((feature) => feature.name === 'embed')) {\n    const embeddingsTable = await conn.prepare(\"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'embeddings'\");\n    if ((await embeddingsTable.all()).length > 0) await withTransaction(conn, () => conn.exec('DELETE FROM embeddings'), dialect.beginMode());\n  }\n\n  const forcedPaths = new Set(listFiles(cfg, paths.rootDir).map((file) => file.relPath));\n  return reconcile(conn, cfg, paths, dialect, pool, forcedPaths);\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, paths: ReconcilePaths, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, paths, dialect, pool, forcedPaths),\n    recover: () => recoverIncomplete(conn, cfg, paths, dialect, pool),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, paths.rootDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, paths.rootDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["featureEnabled","embed","FEATURES","rank","listFiles","ParsePool","reparseFiles","NARROW_FEATURE_TABLE","reconcile","getColumns","withTransaction","nullEmbedVectors","conn","dialect","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","prepare","existingPaths","Set","all","map","r","path","files","filter","f","has","relPath","length","docs","undefined","paths","d","delta","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","name","feature","find","Error","turnedOn","docsForFeature","afterReconcile","nullRank","columns","rerunRank","invalidateFeatureToggles","toggles","byName","Map","linksToggle","get","rankToggle","recoverIncomplete","disabled","schema","rootDir","some","embeddingsTable","forcedPaths","file","createBuilder","build","recover","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":"AAAA,SAAsBA,cAAc,QAAQ,qBAAqB;AACjE,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,IAAI,QAAQ,sBAAsB;AAE3C,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,YAAY,QAAQ,qBAAqB;AAGlD,SAASC,oBAAoB,QAAQ,qBAAqB;AAC1D,SAA8BC,SAAS,QAAQ,iBAAiB;AAChE,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,eAAe,QAAQ,mBAAmB;AAwBnD,6FAA6F;AAC7F,wEAAwE;AACxE,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;IACzE,MAAMH,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,sDAAsDD,QAAQE,SAAS;IACnH,OAAO;QAAEC,QAAQ;IAAE;AACrB;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,eAAeC,kBAAkBL,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe;IACzH,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,gBAAgB,IAAIC,IAAI,AAAC,CAAA,MAAMH,aAAaI,GAAG,EAAC,EAAGC,GAAG,CAAC,CAACC,IAAM,AAACA,EAAuBC,IAAI;IAChG,8FAA8F;IAC9F,4FAA4F;IAC5F,MAAMC,QAAQzB,UAAUc,KAAKC,SAASW,MAAM,CAAC,CAACC,IAAMA,EAAE9B,KAAK,IAAIsB,cAAcS,GAAG,CAACD,EAAEE,OAAO;IAC1F,IAAIJ,MAAMK,MAAM,KAAK,GAAG,OAAO;QAAElB,QAAQ;IAAE;IAE3C,MAAM,EAAEmB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAAC5B;KAAM,EAAEiB,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACtF,MAAMiB,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IAChF,MAAMhC,gBACJE,MACA;YACQX,eACAA;QADN,QAAMA,gBAAAA,MAAM0C,MAAM,cAAZ1C,oCAAAA,mBAAAA,OAAeW,MAAMyB,OAAOE;QAClC,QAAMtC,eAAAA,MAAM2C,KAAK,cAAX3C,mCAAAA,kBAAAA,OACJW,MACAuB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;gBAAEV,MAAMU,EAAEL,OAAO;gBAAEY,WAAWP,EAAEO,SAAS,CAAC5C,KAAK;YAAC,CAAA,IACjEsC;IAEJ,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,eAAeY,uBAAuBlC,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAE2B,MAAqB;IACrJ,MAAMC,QAAQzC,oBAAoB,CAACwC,OAAOE,IAAI,CAAC;IAC/C,MAAMC,UAAUhD,SAASiD,IAAI,CAAC,CAACpB,IAAMA,EAAEkB,IAAI,KAAKF,OAAOE,IAAI;IAC3D,IAAI,CAACC,SAAS,MAAM,IAAIE,MAAM,CAAC,qDAAqD,EAAEL,OAAOE,IAAI,CAAC,CAAC,CAAC;IAEpG,IAAI,CAACF,OAAOM,QAAQ,EAAE;QACpB,MAAM3C,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO,GAAGnC,QAAQE,SAAS;QACtF,OAAO;YAAEC,QAAQ;QAAE;IACrB;IAEA,MAAMa,QAAQzB,UAAUc,KAAKC;IAC7B,MAAM,EAAEgB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAACqB;KAAQ,EAAEhC,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACxF,MAAMkC,iBAAiBnB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;YAAEV,MAAMU,EAAEL,OAAO;YAAEY,WAAWP,EAAEO,SAAS,CAACK,QAAQD,IAAI,CAAC;QAAC,CAAA;IAChG,MAAMZ,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,8FAA8F;IAC9F,kFAAkF;IAClF,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAOJ;QAAOK,UAAU,EAAE;IAAC;IACnF,MAAMhC,gBACJE,MACA;YAEQsC,gBACAA;QAFN,MAAMtC,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO;QACtC,QAAME,iBAAAA,QAAQN,KAAK,cAAbM,qCAAAA,oBAAAA,SAAgBtC,MAAM0C,gBAAgBf;QAC5C,QAAMW,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyBtC,MAAM2B;IACvC,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,eAAesB,SAAS5C,IAAgB,EAAEC,OAAyB;IACjE,MAAM4C,UAAU,MAAMhD,WAAWG;IACjC,IAAI,CAAC6C,QAAQzB,GAAG,CAAC,UAAU;IAC3B,MAAMtB,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,0CAA0CD,QAAQE,SAAS;AACzG;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,eAAe2C,UAAU9C,IAAgB,EAAEC,OAAyB;IAClE,MAAM0B,QAAwB;QAAEV,OAAO,EAAE;QAAEW,UAAU,EAAE;QAAEC,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IACjF,MAAMhC,gBACJE,MACA;YACQT;QAAN,QAAMA,uBAAAA,KAAKoD,cAAc,cAAnBpD,2CAAAA,0BAAAA,MAAsBS,MAAM2B;IACpC,GACA1B,QAAQE,SAAS;AAErB;AAEA,eAAe4C,yBAAyB/C,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAEwC,OAAwB;IAC1J,MAAMC,SAAS,IAAIC,IAAIF,QAAQlC,GAAG,CAAC,CAACqB,SAAW;YAACA,OAAOE,IAAI;YAAEF;SAAO;IACpE,IAAI/B,SAAS;IAEb,4FAA4F;IAC5F,mFAAmF;IACnF,MAAM+C,cAAcF,OAAOG,GAAG,CAAC;IAC/B,IAAID,aAAa;QACf/C,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2C,YAAW,EAAG/C,MAAM;QAC/F,4FAA4F;QAC5F,0FAA0F;QAC1F,IAAI,CAAC+C,YAAYV,QAAQ,EAAE,MAAMG,SAAS5C,MAAMC;IAClD;IAEA,wFAAwF;IACxF,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMoD,aAAaJ,OAAOG,GAAG,CAAC;IAC9B,IAAIC,YAAY;QACd,IAAIA,WAAWZ,QAAQ,EAAE,MAAMK,UAAU9C,MAAMC;aAC1C,MAAM2C,SAAS5C,MAAMC;IAC5B;IAEA,KAAK,MAAMkC,UAAUa,QAAS;QAC5B,IAAIb,OAAOE,IAAI,KAAK,WAAWF,OAAOE,IAAI,KAAK,QAAQ;QACvDjC,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2B,OAAM,EAAG/B,MAAM;IAC5F;IACA,OAAO;QAAEA;IAAO;AAClB;AAEA,eAAekD,kBAAkBtD,IAAgB,EAAEM,GAAW,EAAEmB,KAAqB,EAAExB,OAAyB,EAAEO,IAAe;IAC/H,MAAM+C,WAAWjE,SAAS4B,MAAM,CAAC,CAACoB,UAAY,CAAClD,eAAekB,KAAKgC,QAAQD,IAAI;IAC/E,KAAK,MAAMC,WAAWiB,SAAU;QAC9B,IAAIjB,QAAQD,IAAI,KAAK,SAAS,MAAMC,QAAQkB,MAAM,CAACxD;IACrD;IAEA,MAAMgD,UAA2BO,SAASrC,MAAM,CAAC,CAACoB,UAAYA,QAAQD,IAAI,KAAK,SAASvB,GAAG,CAAC,CAACwB,UAAa,CAAA;YAAED,MAAMC,QAAQD,IAAI;YAAEI,UAAU;QAAM,CAAA;IAChJ,MAAMM,yBAAyB/C,MAAMM,KAAKmB,MAAMgC,OAAO,EAAExD,SAASO,MAAMwC;IACxE,IAAIO,SAASG,IAAI,CAAC,CAACpB,UAAYA,QAAQD,IAAI,KAAK,UAAU;QACxD,MAAMsB,kBAAkB,MAAM3D,KAAKU,OAAO,CAAC;QAC3C,IAAI,AAAC,CAAA,MAAMiD,gBAAgB9C,GAAG,EAAC,EAAGS,MAAM,GAAG,GAAG,MAAMxB,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,2BAA2BD,QAAQE,SAAS;IACxI;IAEA,MAAMyD,cAAc,IAAIhD,IAAIpB,UAAUc,KAAKmB,MAAMgC,OAAO,EAAE3C,GAAG,CAAC,CAAC+C,OAASA,KAAKxC,OAAO;IACpF,OAAOzB,UAAUI,MAAMM,KAAKmB,OAAOxB,SAASO,MAAMoD;AACpD;AAEA,4FAA4F;AAC5F,6DAA6D;AAC7D,OAAO,SAASE,cAAc9D,IAAgB,EAAEM,GAAW,EAAEmB,KAAqB,EAAExB,OAAyB;IAC3G,MAAMO,OAAO,IAAIf;IACjB,OAAO;QACLsE,OAAO,CAACH,cAAgBhE,UAAUI,MAAMM,KAAKmB,OAAOxB,SAASO,MAAMoD;QACnEI,SAAS,IAAMV,kBAAkBtD,MAAMM,KAAKmB,OAAOxB,SAASO;QAC5DyD,YAAY,CAACC,OAAUA,SAAS,UAAUnE,iBAAiBC,MAAMC,WAAWI,kBAAkBL,MAAMM,KAAKmB,MAAMgC,OAAO,EAAExD,SAASO;QACjI2D,oBAAoB,CAACnB,UAAYD,yBAAyB/C,MAAMM,KAAKmB,MAAMgC,OAAO,EAAExD,SAASO,MAAMwC;QACnGoB,OAAO,IAAM5D,KAAK4D,KAAK;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}

package/dist/esm/store/duckdb/lexical.d.ts

@@ -4,6 +4,8 @@ interface FtsIndexState {
     orderedMacroReady: boolean;
 }
 export declare function markContentStale(conn: Connection): Promise<void>;
+export declare function prepareFts(conn: Connection): Promise<void>;
+export declare function assertFtsReady(conn: Connection): Promise<void>;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;

package/dist/esm/store/duckdb/lexical.js

@@ -75,6 +75,17 @@ async function stateFor(conn) {
     }
     return state;
 }
+async function hasFtsArtifact(conn) {
+    const stmt = await conn.prepare(`
+    SELECT 1 AS ready
+    FROM duckdb_functions()
+    WHERE schema_name = 'fts_main_content'
+      AND function_name = 'match_bm25'
+      AND function_type = 'macro'
+    LIMIT 1
+  `);
+    return await stmt.get() !== undefined;
+}
 // Called whenever this store's reconcileContent changes `content`; must run inside that same
 // transaction (reconcile.ts) so a crash never lands the content write without the stale mark.
 export async function markContentStale(conn) {
@@ -131,10 +142,11 @@ function splitTerms(terms) {
 }
 // No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays
 // the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.
-async function ensureFtsFresh(conn, state) {
-    if (!state.stale) return;
-    state.orderedMacroReady = false;
+async function rebuildFts(conn, state) {
     await conn.exec('INSTALL fts; LOAD fts;');
+    if (!state.stale && await hasFtsArtifact(conn)) return;
+    state.stale = true;
+    state.orderedMacroReady = false;
     await withTransaction(conn, async ()=>{
         // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
         // default 571-word list would otherwise drop common words (e.g. "and") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.
@@ -145,6 +157,22 @@ async function ensureFtsFresh(conn, state) {
     });
     state.stale = false;
 }
+export async function prepareFts(conn) {
+    await rebuildFts(conn, await stateFor(conn));
+}
+export async function assertFtsReady(conn) {
+    const state = await stateFor(conn);
+    if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');
+    try {
+        await conn.exec('LOAD fts;');
+        if (!await hasFtsArtifact(conn)) {
+            throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');
+        }
+    } catch (err) {
+        if (err instanceof SenseError) throw err;
+        throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${err.message}); run \`sense build\` (or library build(config))`);
+    }
+}
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
 function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
@@ -199,7 +227,7 @@ export async function queryLexical(conn, terms, opts, state) {
     const { whereJoin, whereCond, scopeCond, limit } = opts;
     const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);
     if (emptyPhrase || words.length === 0 && substrings.length === 0) return [];
-    if (words.length > 0) await ensureFtsFresh(conn, state);
+    if (words.length > 0) await assertFtsReady(conn);
     if (words.length > 0) await ensureOrderedBm25(conn, state);
     const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);
     const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  state.orderedMacroReady = false;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","ensureOrderedBm25","ORDERED_BM25_MACRO","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","orderedMacroReady","set","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","stmt","prepare","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAE1E,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM5B,QAAQyB,MAAMJ,wBAAyB;YAAKQ,mBAAmB;QAAM;QAC7FP,SAASQ,GAAG,CAACL,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeK,iBAAiBN,IAAgB;IACrDH,SAASQ,GAAG,CAACL,MAAM;QAAEG,OAAO;QAAMC,mBAAmB;IAAM;IAC3D,MAAM5B,QAAQwB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASW,WAAWlB,KAAa;IAC/B,MAAMmB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAMvB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACuB,IAAIC;QACtD,MAAMC,SAASD,MAAMpB,IAAI;QACzB,MAAMsB,eAAe3C,aAAa0C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAMvB,SAASuB,aAAc;YAChC,IAAIvB,MAAM0B,QAAQ,EAAET,WAAWQ,IAAI,CAACzB,MAAMV,IAAI;iBACzC;gBACHwB,MAAMW,IAAI,CAACzB,MAAMV,IAAI;gBACrB0B,YAAYS,IAAI,CAACzB,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMqC,OAAO/B,eAAegC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAMxB,SAASpB,aAAa+C,KAAM;YACrC,IAAI3B,MAAM0B,QAAQ,EAAET,WAAWQ,IAAI,CAACzB,MAAMV,IAAI;iBACzC;gBACHwB,MAAMW,IAAI,CAACzB,MAAMV,IAAI;gBACrByB,UAAUU,IAAI,CAACzB,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAEwB;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,eAAevB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClBF,MAAMG,iBAAiB,GAAG;IAC1B,MAAMJ,KAAKwB,IAAI,CAAC;IAChB,MAAM/C,gBAAgBuB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKwB,IAAI,CAAC,CAAC,qGAAqG,CAAC;QACvH,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMhD,QAAQwB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASsB,kBAAkBjB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMU,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYtB,MAAMuB,GAAG,CAAC3D,eAAe4D,IAAI,CAAC;QAChD,KAAK,MAAMC,SAASrD,OAAQ;YAC1B8C,WAAWP,IAAI,CAAC,GAAGtC,YAAY,CAACoD,MAAM,CAAC,cAAc,EAAEtD,mBAAmB,+BAA+B,EAAEsD,MAAM,4BAA4B,CAAC;YAC9IL,YAAYT,IAAI,CAACW;QACnB;QACA,IAAIrB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMgB,aAAa,GAAGvD,mBAAmB,oDAAoD,CAAC;YAC9FgD,UAAUR,IAAI,CAACe;YACfL,WAAWV,IAAI,CAACV,UAAUsB,GAAG,CAAC3D,eAAe4D,IAAI,CAAC;QACpD;QACA,IAAItB,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMgB,aAAa,GAAGvD,mBAAmB,oDAAoD,CAAC;YAC9F,MAAMwD,WAAWzB,YAAYqB,GAAG,CAAC,CAACK,QAAU,CAAC,CAAC,EAAExD,OAAOmD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJL,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEe,WAAW,KAAK,EAAEC,SAAS,EAAE,CAAC;YACjDN,WAAWV,IAAI,CAACT,YAAYqB,GAAG,CAAC3D,eAAe4D,IAAI,CAAC;YACpDH,WAAWV,IAAI,IAAIT,YAAY2B,OAAO,CAAC,CAACC,OAAS1D,OAAOmD,GAAG,CAAC,IAAM3D,cAAckE;QAClF;IACF;IAEA,KAAK,MAAMC,OAAO5B,WAAY;QAC5B,MAAM6B,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMR,SAASrD,OAAQ;YAC1B8C,WAAWP,IAAI,CAAC,GAAGtC,YAAY,CAACoD,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHL,YAAYT,IAAI,CAAC/C,cAAcoE;QACjC;QACAb,UAAUR,IAAI,CAAC,CAAC,CAAC,EAAEvC,OAAOmD,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GH,WAAWV,IAAI,IAAIvC,OAAOmD,GAAG,CAAC,IAAM3D,cAAcoE;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWM,IAAI,CAAC;QAAQW,SAAShB,UAAUK,IAAI,CAAC,YAAY;QAAQY,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAa7C,IAAgB,EAAEX,KAAa,EAAEyD,IAAyB,EAAE7C,KAAoB;IACjH,MAAM8C,cAAc3D,oBAAoBC;IACxC,IAAI0D,gBAAgB,MAAM;QACxB,MAAM,IAAI5E,WAAW,4BAA4B,CAAC,yCAAyC,EAAE4E,YAAY7D,KAAK,CAAC,GAAG,EAAE6D,YAAYrD,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEqD,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAEtC,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWlB;IACvF,IAAIwB,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMK,eAAevB,MAAMC;IACjD,IAAIO,MAAMU,MAAM,GAAG,GAAG,MAAMxC,kBAAkBsB,MAAMC;IAEpD,MAAM,EAAEyC,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBjB,OAAOC,WAAWC,aAAaC;IACvF,MAAMyC,eAAexC,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMmC,WAAWzC,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMoC,MAAM,CAAC,kBAAkB,EAAE1C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAEwB,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAME,OAAO,MAAMvD,KAAKwD,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,IAAId,WAAYhC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACiC;KAAM;IAC7E,MAAMQ,UAAU/C,QAAQM,MAAM,KAAK,IAAIuC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQjD,QAAQkD,KAAK,CAAC,CAAC9C,SAAWpC,OAAOmF,IAAI,CAAC,CAAC9B;oBAA8B4B;uBAApBxF,qBAAoBwF,aAAAA,GAAG,CAAC5B,MAAM,cAAV4B,wBAAAA,aAAc,IAAI7C;;IAC1J,OAAO2C,QAAQK,KAAK,CAAC,GAAGb,OAAOpB,GAAG,CAAC,CAAC,EAAEkC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBlE,IAAgB;IACjD,OAAO;QACLmE,OAAO,OAAO9E,OAAOyD,OAASD,aAAa7C,MAAMX,OAAOyD,MAAM,MAAM/C,SAASC;QAC7EoE,WAAW,IAAM9D,iBAAiBN;IACpC;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS snippets for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { foldForSearch, matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { ensureOrderedBm25, ORDERED_BM25_MACRO } from './ordered-bm25.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Quoted contents stay ignored while their boundary remains detectable.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n  { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n  orderedMacroReady: boolean;\n}\n\n// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of\n// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.\nconst FTS_STALE_META_KEY = 'fts_stale';\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nasync function stateFor(conn: Connection): Promise<FtsIndexState> {\n  let state = ftsState.get(conn);\n  if (!state) {\n    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that\n    // never built an index, or one a prior process marked stale and never got to clear.\n    state = { stale: (await getMeta(conn, FTS_STALE_META_KEY)) !== '0', orderedMacroReady: false };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\nasync function hasFtsArtifact(conn: Connection): Promise<boolean> {\n  const stmt = await conn.prepare(`\n    SELECT 1 AS ready\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content'\n      AND function_name = 'match_bm25'\n      AND function_type = 'macro'\n    LIMIT 1\n  `);\n  return (await stmt.get()) !== undefined;\n}\n\n// Called whenever this store's reconcileContent changes `content`; must run inside that same\n// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.\nexport async function markContentStale(conn: Connection): Promise<void> {\n  ftsState.set(conn, { stale: true, orderedMacroReady: false });\n  await setMeta(conn, FTS_STALE_META_KEY, '1');\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,\n// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.\nfunction splitTerms(terms: string): { words: string[]; bareWords: string[]; phraseWords: string[]; substrings: string[]; phrases: SearchToken[][]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const bareWords: string[] = [];\n  const phraseWords: string[] = [];\n  const substrings: string[] = [];\n  const phrases: SearchToken[][] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const phraseTokens = searchTokens(phrase);\n    if (phraseTokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    phrases.push(phraseTokens);\n    for (const token of phraseTokens) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        phraseWords.push(token.text);\n      }\n    }\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) substrings.push(token.text);\n      else {\n        words.push(token.text);\n        bareWords.push(token.text);\n      }\n    }\n  }\n  return { words, bareWords, phraseWords, substrings, phrases, emptyPhrase };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays\n// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.\nasync function rebuildFts(conn: Connection, state: FtsIndexState): Promise<void> {\n  await conn.exec('INSTALL fts; LOAD fts;');\n  if (!state.stale && (await hasFtsArtifact(conn))) return;\n  state.stale = true;\n  state.orderedMacroReady = false;\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n    // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant\n    // rebuild next time), never a cleared mark serving a half-built index as fresh.\n    await setMeta(conn, FTS_STALE_META_KEY, '0');\n  });\n  state.stale = false;\n}\n\nexport async function prepareFts(conn: Connection): Promise<void> {\n  await rebuildFts(conn, await stateFor(conn));\n}\n\nexport async function assertFtsReady(conn: Connection): Promise<void> {\n  const state = await stateFor(conn);\n  if (state.stale) throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index is missing or stale; run `sense build` (or library build(config)) before lexical querying with build disabled');\n  try {\n    await conn.exec('LOAD fts;');\n    if (!(await hasFtsArtifact(conn))) {\n      throw new SenseError('INDEX_NOT_READY', 'the DuckDB lexical index artifact is missing; run `sense build` (or library build(config)) before lexical querying with build disabled');\n    }\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('INDEX_NOT_READY', `the DuckDB lexical extension is not prepared (${(err as Error).message}); run \\`sense build\\` (or library build(config))`);\n  }\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], bareWords: string[], phraseWords: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.map(foldForSearch).join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(${ORDERED_BM25_MACRO}(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    if (bareWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      gateParts.push(nativeGate);\n      gateParams.push(bareWords.map(foldForSearch).join(' '));\n    }\n    if (phraseWords.length > 0) {\n      const nativeGate = `${ORDERED_BM25_MACRO}(content.\"path\", ?, conjunctive := true) IS NOT NULL`;\n      const fallback = phraseWords.map((_word) => `(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');\n      gateParts.push(`(${nativeGate} OR (${fallback}))`);\n      gateParams.push(phraseWords.map(foldForSearch).join(' '));\n      gateParams.push(...phraseWords.flatMap((word) => FIELDS.map(() => foldForSearch(word))));\n    }\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);\n      scoreParams.push(foldForSearch(needle));\n    }\n    gateParts.push(`(${FIELDS.map((field) => `contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);\n    gateParams.push(...FIELDS.map(() => foldForSearch(needle)));\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND ') || 'TRUE', params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase || (words.length === 0 && substrings.length === 0)) return [];\n\n  if (words.length > 0) await assertFtsReady(conn);\n  if (words.length > 0) await ensureOrderedBm25(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);\n  const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';\n  const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score${selectFields}\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n    ) sq ORDER BY score DESC, path${sqlLimit}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...params, ...(phrases.length > 0 ? [] : [limit]))) as Array<LexicalHit & Partial<Record<Field, string>>>;\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, limit).map(({ path }) => ({ path }));\n}\n\n// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know\n// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => Promise<void> } {\n  return {\n    query: async (terms, opts) => queryLexical(conn, terms, opts, await stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","foldForSearch","matchesSearchPhrase","searchTokens","getMeta","setMeta","withTransaction","ensureOrderedBm25","ORDERED_BM25_MACRO","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","FTS_STALE_META_KEY","ftsState","WeakMap","stateFor","conn","state","get","stale","orderedMacroReady","set","hasFtsArtifact","stmt","prepare","undefined","markContentStale","splitTerms","words","bareWords","phraseWords","substrings","phrases","emptyPhrase","_m","inner","phrase","phraseTokens","length","push","unspaced","tok","split","rebuildFts","exec","prepareFts","assertFtsReady","err","message","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","map","join","field","nativeGate","fallback","_word","flatMap","word","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","selectFields","sqlLimit","sql","rows","all","matches","filter","row","every","some","slice","path","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,aAAa,EAAEC,mBAAmB,EAAoBC,YAAY,QAAQ,wBAAwB;AAC3G,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,iBAAiB,EAAEC,kBAAkB,QAAQ,oBAAoB;AAE1E,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,8HAA8H;AAC9H,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAOA,kGAAkG;AAClG,kGAAkG;AAClG,MAAMC,qBAAqB;AAE3B,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,eAAeC,SAASC,IAAgB;IACtC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACV,2FAA2F;QAC3F,oFAAoF;QACpFA,QAAQ;YAAEE,OAAO,AAAC,MAAM5B,QAAQyB,MAAMJ,wBAAyB;YAAKQ,mBAAmB;QAAM;QAC7FP,SAASQ,GAAG,CAACL,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,eAAeK,eAAeN,IAAgB;IAC5C,MAAMO,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC;;;;;;;EAOjC,CAAC;IACD,OAAO,AAAC,MAAMD,KAAKL,GAAG,OAAQO;AAChC;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,iBAAiBV,IAAgB;IACrDH,SAASQ,GAAG,CAACL,MAAM;QAAEG,OAAO;QAAMC,mBAAmB;IAAM;IAC3D,MAAM5B,QAAQwB,MAAMJ,oBAAoB;AAC1C;AAEA,wHAAwH;AACxH,8FAA8F;AAC9F,SAASe,WAAWtB,KAAa;IAC/B,MAAMuB,QAAkB,EAAE;IAC1B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAwB,EAAE;IAChC,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,UAA2B,EAAE;IACnC,IAAIC,cAAc;IAClB,MAAM3B,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAAC2B,IAAIC;QACtD,MAAMC,SAASD,MAAMxB,IAAI;QACzB,MAAM0B,eAAe/C,aAAa8C;QAClC,IAAIC,aAAaC,MAAM,KAAK,GAAG;YAC7BL,cAAc;YACd,OAAO;QACT;QACAD,QAAQO,IAAI,CAACF;QACb,KAAK,MAAM3B,SAAS2B,aAAc;YAChC,IAAI3B,MAAM8B,QAAQ,EAAET,WAAWQ,IAAI,CAAC7B,MAAMV,IAAI;iBACzC;gBACH4B,MAAMW,IAAI,CAAC7B,MAAMV,IAAI;gBACrB8B,YAAYS,IAAI,CAAC7B,MAAMV,IAAI;YAC7B;QACF;QACA,OAAO;IACT;IACA,KAAK,MAAMyC,OAAOnC,eAAeoC,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIH,MAAM,KAAK,GAAG;QACtB,KAAK,MAAM5B,SAASpB,aAAamD,KAAM;YACrC,IAAI/B,MAAM8B,QAAQ,EAAET,WAAWQ,IAAI,CAAC7B,MAAMV,IAAI;iBACzC;gBACH4B,MAAMW,IAAI,CAAC7B,MAAMV,IAAI;gBACrB6B,UAAUU,IAAI,CAAC7B,MAAMV,IAAI;YAC3B;QACF;IACF;IACA,OAAO;QAAE4B;QAAOC;QAAWC;QAAaC;QAAYC;QAASC;IAAY;AAC3E;AAEA,gGAAgG;AAChG,iHAAiH;AACjH,eAAeU,WAAW3B,IAAgB,EAAEC,KAAoB;IAC9D,MAAMD,KAAK4B,IAAI,CAAC;IAChB,IAAI,CAAC3B,MAAME,KAAK,IAAK,MAAMG,eAAeN,OAAQ;IAClDC,MAAME,KAAK,GAAG;IACdF,MAAMG,iBAAiB,GAAG;IAC1B,MAAM3B,gBAAgBuB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAK4B,IAAI,CAAC,CAAC,qGAAqG,CAAC;QACvH,0FAA0F;QAC1F,gFAAgF;QAChF,MAAMpD,QAAQwB,MAAMJ,oBAAoB;IAC1C;IACAK,MAAME,KAAK,GAAG;AAChB;AAEA,OAAO,eAAe0B,WAAW7B,IAAgB;IAC/C,MAAM2B,WAAW3B,MAAM,MAAMD,SAASC;AACxC;AAEA,OAAO,eAAe8B,eAAe9B,IAAgB;IACnD,MAAMC,QAAQ,MAAMF,SAASC;IAC7B,IAAIC,MAAME,KAAK,EAAE,MAAM,IAAIhC,WAAW,mBAAmB;IACzD,IAAI;QACF,MAAM6B,KAAK4B,IAAI,CAAC;QAChB,IAAI,CAAE,MAAMtB,eAAeN,OAAQ;YACjC,MAAM,IAAI7B,WAAW,mBAAmB;QAC1C;IACF,EAAE,OAAO4D,KAAK;QACZ,IAAIA,eAAe5D,YAAY,MAAM4D;QACrC,MAAM,IAAI5D,WAAW,mBAAmB,CAAC,8CAA8C,EAAE,AAAC4D,IAAcC,OAAO,CAAC,iDAAiD,CAAC;IACpK;AACF;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASC,kBAAkBrB,KAAe,EAAEC,SAAmB,EAAEC,WAAqB,EAAEC,UAAoB;IAC1G,MAAMmB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIzB,MAAMU,MAAM,GAAG,GAAG;QACpB,MAAMgB,YAAY1B,MAAM2B,GAAG,CAACnE,eAAeoE,IAAI,CAAC;QAChD,KAAK,MAAMC,SAAS7D,OAAQ;YAC1BsD,WAAWX,IAAI,CAAC,GAAG1C,YAAY,CAAC4D,MAAM,CAAC,cAAc,EAAE9D,mBAAmB,+BAA+B,EAAE8D,MAAM,4BAA4B,CAAC;YAC9IL,YAAYb,IAAI,CAACe;QACnB;QACA,IAAIzB,UAAUS,MAAM,GAAG,GAAG;YACxB,MAAMoB,aAAa,GAAG/D,mBAAmB,oDAAoD,CAAC;YAC9FwD,UAAUZ,IAAI,CAACmB;YACfL,WAAWd,IAAI,CAACV,UAAU0B,GAAG,CAACnE,eAAeoE,IAAI,CAAC;QACpD;QACA,IAAI1B,YAAYQ,MAAM,GAAG,GAAG;YAC1B,MAAMoB,aAAa,GAAG/D,mBAAmB,oDAAoD,CAAC;YAC9F,MAAMgE,WAAW7B,YAAYyB,GAAG,CAAC,CAACK,QAAU,CAAC,CAAC,EAAEhE,OAAO2D,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAEA,IAAI,CAAC;YACnJL,UAAUZ,IAAI,CAAC,CAAC,CAAC,EAAEmB,WAAW,KAAK,EAAEC,SAAS,EAAE,CAAC;YACjDN,WAAWd,IAAI,CAACT,YAAYyB,GAAG,CAACnE,eAAeoE,IAAI,CAAC;YACpDH,WAAWd,IAAI,IAAIT,YAAY+B,OAAO,CAAC,CAACC,OAASlE,OAAO2D,GAAG,CAAC,IAAMnE,cAAc0E;QAClF;IACF;IAEA,KAAK,MAAMC,OAAOhC,WAAY;QAC5B,MAAMiC,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMR,SAAS7D,OAAQ;YAC1BsD,WAAWX,IAAI,CAAC,GAAG1C,YAAY,CAAC4D,MAAM,CAAC,+CAA+C,EAAEA,MAAM,kBAAkB,CAAC;YACjHL,YAAYb,IAAI,CAACnD,cAAc4E;QACjC;QACAb,UAAUZ,IAAI,CAAC,CAAC,CAAC,EAAE3C,OAAO2D,GAAG,CAAC,CAACE,QAAU,CAAC,qCAAqC,EAAEA,MAAM,MAAM,CAAC,EAAED,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/GH,WAAWd,IAAI,IAAI3C,OAAO2D,GAAG,CAAC,IAAMnE,cAAc4E;IACpD;IAEA,OAAO;QAAEE,UAAUhB,WAAWM,IAAI,CAAC;QAAQW,SAAShB,UAAUK,IAAI,CAAC,YAAY;QAAQY,QAAQ;eAAIhB;eAAgBC;SAAW;IAAC;AACjI;AAEA,uGAAuG;AACvG,0GAA0G;AAC1G,OAAO,eAAegB,aAAarD,IAAgB,EAAEX,KAAa,EAAEiE,IAAyB,EAAErD,KAAoB;IACjH,MAAMsD,cAAcnE,oBAAoBC;IACxC,IAAIkE,gBAAgB,MAAM;QACxB,MAAM,IAAIpF,WAAW,4BAA4B,CAAC,yCAAyC,EAAEoF,YAAYrE,KAAK,CAAC,GAAG,EAAEqE,YAAY7D,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAE6D,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE1C,KAAK,EAAEC,SAAS,EAAEC,WAAW,EAAEC,UAAU,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGN,WAAWtB;IACvF,IAAI4B,eAAgBL,MAAMU,MAAM,KAAK,KAAKP,WAAWO,MAAM,KAAK,GAAI,OAAO,EAAE;IAE7E,IAAIV,MAAMU,MAAM,GAAG,GAAG,MAAMQ,eAAe9B;IAC3C,IAAIY,MAAMU,MAAM,GAAG,GAAG,MAAM5C,kBAAkBsB,MAAMC;IAEpD,MAAM,EAAEiD,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGnB,kBAAkBrB,OAAOC,WAAWC,aAAaC;IACvF,MAAM6C,eAAe5C,QAAQM,MAAM,GAAG,IAAI,mDAAmD;IAC7F,MAAMuC,WAAW7C,QAAQM,MAAM,GAAG,IAAI,KAAK;IAC3C,MAAMwC,MAAM,CAAC,kBAAkB,EAAE9C,QAAQM,MAAM,GAAG,IAAI,2BAA2B,GAAG;oCAClD,EAAE4B,SAAS,UAAU,EAAEU,aAAa;;IAEpE,EAAEJ,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;kCACK,EAAEG,UAAU;IAC5C,MAAMtD,OAAO,MAAMP,KAAKQ,OAAO,CAACsD;IAChC,MAAMC,OAAQ,MAAMxD,KAAKyD,GAAG,IAAIZ,WAAYpC,QAAQM,MAAM,GAAG,IAAI,EAAE,GAAG;QAACqC;KAAM;IAC7E,MAAMM,UAAUjD,QAAQM,MAAM,KAAK,IAAIyC,OAAOA,KAAKG,MAAM,CAAC,CAACC,MAAQnD,QAAQoD,KAAK,CAAC,CAAChD,SAAWxC,OAAOyF,IAAI,CAAC,CAAC5B;oBAA8B0B;uBAApB9F,qBAAoB8F,aAAAA,GAAG,CAAC1B,MAAM,cAAV0B,wBAAAA,aAAc,IAAI/C;;IAC1J,OAAO6C,QAAQK,KAAK,CAAC,GAAGX,OAAOpB,GAAG,CAAC,CAAC,EAAEgC,IAAI,EAAE,GAAM,CAAA;YAAEA;QAAK,CAAA;AAC3D;AAEA,iGAAiG;AACjG,gGAAgG;AAChG,OAAO,SAASC,mBAAmBxE,IAAgB;IACjD,OAAO;QACLyE,OAAO,OAAOpF,OAAOiE,OAASD,aAAarD,MAAMX,OAAOiE,MAAM,MAAMvD,SAASC;QAC7E0E,WAAW,IAAMhE,iBAAiBV;IACpC;AACF"}

package/dist/esm/store/duckdb/open.d.ts

@@ -1,13 +1,13 @@
 import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { OpenResult } from '../open.js';
+import type { InternalOpenOptions, OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 interface DuckdbHandle {
     instance: DuckDBInstance;
     duckdb: DuckDBConnection;
 }
 export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
-export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult>;

package/dist/esm/store/duckdb/open.js

@@ -5,6 +5,7 @@ import { activeFeatures, FEATURES } from '../../features/index.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
+import { assertFtsReady, prepareFts } from './lexical.js';
 import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
 import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
@@ -13,12 +14,14 @@ import { createVectorWriteStage } from './vectors.js';
 export const DB_FILENAME = 'cache.duckdb';
 // Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).
 // The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.
-export const SCHEMA_VERSION = '4';
-// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
+export const SCHEMA_VERSION = '5';
+// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,
+// while contains() verification reads the table directly.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
 async function ensureSchema(handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" DOUBLE, "_ctime" DOUBLE, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
+    await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources ("path" TEXT PRIMARY KEY, text TEXT NOT NULL)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg)){
@@ -39,7 +42,7 @@ async function close(handle) {
     handle.duckdb.disconnectSync();
     handle.instance.closeSync();
 }
-async function connect(dbPath, _cfg) {
+async function connect(dbPath, _cfg, options) {
     // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports
     // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.
     let DuckDBInstance;
@@ -54,7 +57,9 @@ async function connect(dbPath, _cfg) {
     try {
         // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
         // columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
-        instance = await DuckDBInstance.create(dbPath, {
+        instance = await DuckDBInstance.create(dbPath, options.observational ? {
+            access_mode: 'READ_ONLY'
+        } : {
             storage_compatibility_version: 'v1.5.0'
         });
         duckdb = await instance.connect();
@@ -94,8 +99,10 @@ export const duckdbOpenDialect = {
     // "Cannot open file ... being used by another process". Both are the same condition.
     isLocked: (err)=>/Could not set lock on file|being used by another process/.test(err.message),
     ensureSchema,
+    prepareLexical: prepareFts,
+    assertLexicalReady: assertFtsReady,
     createStore: (handle, conn)=>createStore(handle.instance, handle.duckdb, conn)
 };
-export async function openDuckdb(cfg) {
-    return openWithDialect(cfg, duckdbOpenDialect);
+export async function openDuckdb(cfg, options) {
+    return openWithDialect(cfg, duckdbOpenDialect, options);
 }