sensemaking 0.19.2 → 0.21.0

package/dist/esm/scan/reparse.js

@@ -1,35 +1,10 @@
-import { createRequire } from 'node:module';
 import { availableParallelism } from 'node:os';
-import { dirname, join } from 'node:path';
 import { parseFile } from './index.js';
-import { reviveError } from './worker-error.js';
-// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
-// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.
-const _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;
+import { ParsePool } from './pool.js';
 // Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files
 // (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.
 const DEFAULT_THRESHOLD = 200;
 const DEFAULT_MAX_WORKERS = 8;
-// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
-// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
-let workerFile;
-function resolveWorkerFile() {
-    if (workerFile) return workerFile;
-    const load = createRequire(import.meta.url);
-    for (const rel of [
-        '..',
-        '../..',
-        '../../..'
-    ]){
-        try {
-            if (load(`${rel}/package.json`).name === 'sensemaking') {
-                workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');
-                return workerFile;
-            }
-        } catch  {}
-    }
-    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
-}
 // Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and
 // the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).
 export function featuresForFile(features, cfg, file) {
@@ -45,41 +20,19 @@ function reparseSerial(files, features, cfg, onParsed) {
     }
     return results;
 }
-// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
-// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.
-async function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
-    const { Tinypool } = _require('tinypool');
-    // cfg and the feature selection are constant for the dispatch, so they cross once per worker
-    // as workerData. Features carry closures and cannot cross at all; only their names do.
-    const workerData = {
-        cfg,
-        featureNames: features.map((feature)=>feature.name)
-    };
-    const pool = new Tinypool({
-        filename: resolveWorkerFile(),
-        minThreads: maxWorkers,
-        maxThreads: maxWorkers,
-        workerData
-    });
-    let done = 0;
+// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with
+// none given, opens an ephemeral ParsePool for this call alone and always closes it.
+async function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
+    if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);
+    const ephemeral = new ParsePool();
     try {
-        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
-        // regardless of task completion order, which first-seen column order depends on downstream.
-        const results = await Promise.all(files.map(async (file)=>{
-            const result = await pool.run(file);
-            if (!result.ok) throw reviveError(result.error);
-            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
-            return {
-                doc: result.doc,
-                warnings: result.warnings
-            };
-        }));
-        await pool.destroy();
+        const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);
+        await ephemeral.close();
         return results;
     } catch (err) {
-        // Tear the pool down before rethrowing, but a destroy failure must not mask the parse
+        // Tear the pool down before rethrowing, but a close failure must not mask the parse
         // failure that caused it -- not a `finally`, for that reason.
-        await pool.destroy().catch(()=>{});
+        await ephemeral.close().catch(()=>{});
         throw err;
     }
 }
@@ -89,7 +42,7 @@ export async function reparseFiles(files, features, cfg, knownColumns, onParsed,
     var _options_threshold, _options_maxWorkers;
     const threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : DEFAULT_THRESHOLD;
     const maxWorkers = (_options_maxWorkers = options.maxWorkers) !== null && _options_maxWorkers !== void 0 ? _options_maxWorkers : Math.min(DEFAULT_MAX_WORKERS, availableParallelism());
-    const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);
+    const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);
     const docs = [];
     const warnings = [];
     const newColumns = [];

package/dist/esm/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\n// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n  // as workerData. Features carry closures and cannot cross at all; only their names do.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["createRequire","availableParallelism","dirname","join","parseFile","reviveError","_require","require","url","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","Tinypool","workerData","featureNames","map","pool","filename","minThreads","maxThreads","Promise","all","result","run","ok","error","doc","warnings","destroy","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","newColumns","seen","Set","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,oBAAoB,QAAQ,UAAU;AAC/C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAQ1C,SAASC,SAAS,QAAQ,aAAa;AAEvC,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,qIAAqI;AACrI,MAAMC,WAAW,OAAOC,YAAY,cAAcP,cAAc,YAAYQ,GAAG,IAAID;AAmBnF,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAME,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOb,cAAc,YAAYQ,GAAG;IAC1C,KAAK,MAAMM,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaR,KAAKD,QAAQW,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAAC3B,UAAUiB,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB;IAClJ,MAAM,EAAEC,QAAQ,EAAE,GAAG5B,SAAS;IAC9B,6FAA6F;IAC7F,uFAAuF;IACvF,MAAM6B,aAA8B;QAAEf;QAAKgB,cAAcjB,SAASkB,GAAG,CAAC,CAACd,UAAYA,QAAQR,IAAI;IAAE;IACjG,MAAMuB,OAAO,IAAIJ,SAAS;QAAEK,UAAU3B;QAAqB4B,YAAYP;QAAYQ,YAAYR;QAAYE;IAAW;IACtH,IAAIN,OAAO;IACX,IAAI;QACF,0FAA0F;QAC1F,4FAA4F;QAC5F,MAAMD,UAAU,MAAMc,QAAQC,GAAG,CAC/BjB,MAAMW,GAAG,CAAC,OAAOhB;YACf,MAAMuB,SAAU,MAAMN,KAAKO,GAAG,CAACxB;YAC/B,IAAI,CAACuB,OAAOE,EAAE,EAAE,MAAMzC,YAAYuC,OAAOG,KAAK;YAC9CpB,qBAAAA,+BAAAA,SAAW,EAAEE;YACb,OAAO;gBAAEmB,KAAKJ,OAAOI,GAAG;gBAAEC,UAAUL,OAAOK,QAAQ;YAAC;QACtD;QAEF,MAAMX,KAAKY,OAAO;QAClB,OAAOtB;IACT,EAAE,OAAOuB,KAAK;QACZ,sFAAsF;QACtF,8DAA8D;QAC9D,MAAMb,KAAKY,OAAO,GAAGE,KAAK,CAAC,KAAO;QAClC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAa3B,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEkC,YAAiC,EAAE3B,QAAiC,EAAE4B,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB9C;IACvC,MAAMwB,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAChD,qBAAqBT;IACvE,MAAM2B,UAAUF,MAAMiC,MAAM,IAAIH,YAAY,MAAMxB,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,cAAcR,cAAcC,OAAOP,UAAUC,KAAKO;IAElJ,MAAMiC,OAAoB,EAAE;IAC5B,MAAMX,WAAqB,EAAE;IAC7B,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIT;IACrB,KAAK,MAAM,EAAEN,GAAG,EAAEC,UAAUe,YAAY,EAAE,IAAIpC,QAAS;QACrDqB,SAASlB,IAAI,IAAIiC;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACnB,IAAIoB,IAAI,EAAG;YACvC,IAAI,CAACN,KAAKO,GAAG,CAACJ,MAAM;gBAClBH,KAAKQ,GAAG,CAACL;gBACTJ,WAAW9B,IAAI,CAACkC;YAClB;QACF;QACAL,KAAK7B,IAAI,CAACiB;IACZ;IAEA,OAAO;QAAEY;QAAMX;QAAUY;IAAW;AACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["availableParallelism","parseFile","ParsePool","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","pool","run","ephemeral","close","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","warnings","newColumns","seen","Set","doc","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,oBAAoB,QAAQ,UAAU;AAI/C,SAASC,SAAS,QAAQ,aAAa;AAGvC,SAASC,SAAS,QAAQ,YAAY;AAsBtC,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAMC,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAACjB,UAAUO,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB,EAAEC,IAA2B;IAC/K,IAAIA,MAAM,OAAOA,KAAKC,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;IAC1D,MAAMG,YAAY,IAAIrB;IACtB,IAAI;QACF,MAAMa,UAAU,MAAMQ,UAAUD,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;QACpE,MAAMG,UAAUC,KAAK;QACrB,OAAOT;IACT,EAAE,OAAOU,KAAK;QACZ,oFAAoF;QACpF,8DAA8D;QAC9D,MAAMF,UAAUC,KAAK,GAAGE,KAAK,CAAC,KAAO;QACrC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAad,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEqB,YAAiC,EAAEd,QAAiC,EAAEe,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;IACvC,MAAMiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAC5B,qBAAqBJ;IACvE,MAAMe,UAAUF,MAAMoB,MAAM,IAAIH,YAAY,MAAMX,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,YAAYS,QAAQR,IAAI,IAAIT,cAAcC,OAAOP,UAAUC,KAAKO;IAEhK,MAAMoB,OAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIV;IACrB,KAAK,MAAM,EAAEW,GAAG,EAAEJ,UAAUK,YAAY,EAAE,IAAIzB,QAAS;QACrDoB,SAASjB,IAAI,IAAIsB;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACP,KAAKQ,GAAG,CAACJ,MAAM;gBAClBJ,KAAKS,GAAG,CAACL;gBACTL,WAAWlB,IAAI,CAACuB;YAClB;QACF;QACAP,KAAKhB,IAAI,CAACqB;IACZ;IAEA,OAAO;QAAEL;QAAMC;QAAUC;IAAW;AACtC"}

package/dist/esm/store/builder.d.ts

@@ -0,0 +1,19 @@
+import type { Config } from '../config/index.js';
+import type { EmbedChangeKind } from './embed-scope.js';
+import type { FeatureToggle } from './feature-scope.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export interface Builder {
+    build(forcedPaths?: ReadonlySet<string>): Promise<{
+        parsed: number;
+        warnings: string[];
+    }>;
+    invalidate(kind: EmbedChangeKind): Promise<{
+        parsed: number;
+    }>;
+    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
+        parsed: number;
+    }>;
+    close(): Promise<void>;
+    readonly poolsCreated: number;
+}
+export declare function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder;

package/dist/esm/store/builder.js

@@ -0,0 +1,163 @@
+import { embed } from '../features/embed.js';
+import { FEATURES } from '../features/index.js';
+import { rank } from '../features/rank.js';
+import { listFiles } from '../scan/index.js';
+import { ParsePool } from '../scan/pool.js';
+import { reparseFiles } from '../scan/reparse.js';
+import { NARROW_FEATURE_TABLE } from './feature-scope.js';
+import { reconcile } from './reconcile.js';
+import { getColumns } from './shared.js';
+import { withTransaction } from './transaction.js';
+// 'model': the provider or model name moved but chunk boundaries did not, so only the vector
+// values are stale. No reparse, and no table but embeddings is touched.
+async function nullEmbedVectors(conn, dialect) {
+    await withTransaction(conn, ()=>conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());
+    return {
+        parsed: 0
+    };
+}
+// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds
+// every embedding row through the embed feature's own remove/store, touching no other table.
+async function rebuildEmbeddings(conn, cfg, baseDir, dialect, pool) {
+    const existingStmt = await conn.prepare('SELECT "path" FROM frontmatter');
+    const existingPaths = new Set((await existingStmt.all()).map((r)=>r.path));
+    // A file added since the last build has no row yet; leaving it for build() to insert avoids a
+    // primary-key collision between this INSERT and build()'s own insert for the same new file.
+    const files = listFiles(cfg, baseDir).filter((f)=>f.embed && existingPaths.has(f.relPath));
+    if (files.length === 0) return {
+        parsed: 0
+    };
+    const { docs } = await reparseFiles(files, [
+        embed
+    ], cfg, new Set(), undefined, {
+        pool
+    });
+    const paths = docs.map((d)=>d.relPath);
+    const delta = {
+        files,
+        reparsed: paths,
+        added: [],
+        vanished: []
+    };
+    await withTransaction(conn, async ()=>{
+        var _embed_remove, _embed_store;
+        await ((_embed_remove = embed.remove) === null || _embed_remove === void 0 ? void 0 : _embed_remove.call(embed, conn, paths, delta));
+        await ((_embed_store = embed.store) === null || _embed_store === void 0 ? void 0 : _embed_store.call(embed, conn, docs.map((d)=>({
+                path: d.relPath,
+                extracted: d.extracted.embed
+            })), delta));
+    }, dialect.beginMode());
+    return {
+        parsed: docs.length
+    };
+}
+// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.
+// Rows left over from before this feature was disabled are untrustworthy by construction (files
+// added, edited, or deleted got no remove/store for it), so this never diffs against them --
+// dropping every row and reparsing the whole tree is the only safe path back to consistent state.
+async function invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle) {
+    const table = NARROW_FEATURE_TABLE[toggle.name];
+    const feature = FEATURES.find((f)=>f.name === toggle.name);
+    if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named "${toggle.name}"`);
+    if (!toggle.turnedOn) {
+        await withTransaction(conn, ()=>conn.exec(`DELETE FROM ${table}`), dialect.beginMode());
+        return {
+            parsed: 0
+        };
+    }
+    const files = listFiles(cfg, baseDir);
+    const { docs } = await reparseFiles(files, [
+        feature
+    ], cfg, new Set(), undefined, {
+        pool
+    });
+    const docsForFeature = docs.map((d)=>({
+            path: d.relPath,
+            extracted: d.extracted[feature.name]
+        }));
+    const paths = docs.map((d)=>d.relPath);
+    // Every path counts as added, not just touched: the table was just dropped, so there is nothing
+    // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path
+    // only when added.length equals files.length.
+    const delta = {
+        files,
+        reparsed: paths,
+        added: paths,
+        vanished: []
+    };
+    await withTransaction(conn, async ()=>{
+        var _feature_store, _feature_afterReconcile;
+        await conn.exec(`DELETE FROM ${table}`);
+        await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
+        await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
+    }, dialect.beginMode());
+    return {
+        parsed: docs.length
+    };
+}
+// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column
+// presence, since links can toggle off while rank has never been enabled for this tree.
+async function nullRank(conn, dialect) {
+    const columns = await getColumns(conn);
+    if (!columns.has('_rank')) return;
+    await withTransaction(conn, ()=>conn.exec('UPDATE frontmatter SET "_rank" = NULL'), dialect.beginMode());
+}
+// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so
+// turning rank back on needs no reparseFiles pass at all.
+async function rerunRank(conn, dialect) {
+    const delta = {
+        files: [],
+        reparsed: [],
+        added: [],
+        vanished: []
+    };
+    await withTransaction(conn, async ()=>{
+        var _rank_afterReconcile;
+        await ((_rank_afterReconcile = rank.afterReconcile) === null || _rank_afterReconcile === void 0 ? void 0 : _rank_afterReconcile.call(rank, conn, delta));
+    }, dialect.beginMode());
+}
+async function invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles) {
+    const byName = new Map(toggles.map((toggle)=>[
+            toggle.name,
+            toggle
+        ]));
+    let parsed = 0;
+    // links first, if present: rank's rerun below (whether from its own toggle or the co-change
+    // rankToggle carries) must read the links table after it is back in its new state.
+    const linksToggle = byName.get('links');
+    if (linksToggle) {
+        parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;
+        // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was
+        // never enabled for this tree, so this is safe even when no feature:rank segment changed.
+        if (!linksToggle.turnedOn) await nullRank(conn, dialect);
+    }
+    // rank's effective on/off (featureEnabled's links dependency) only appears as a changed
+    // `feature:rank` segment when it actually flips -- whether that's rank's own config or a
+    // links toggle taking it along -- so honoring it here is complete on its own.
+    const rankToggle = byName.get('rank');
+    if (rankToggle) {
+        if (rankToggle.turnedOn) await rerunRank(conn, dialect);
+        else await nullRank(conn, dialect);
+    }
+    for (const toggle of toggles){
+        if (toggle.name === 'links' || toggle.name === 'rank') continue;
+        parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;
+    }
+    return {
+        parsed
+    };
+}
+// The pool is created at most once, lazily, on whichever build() or invalidate() call first
+// needs it, and reused by every later call on this instance.
+export function createBuilder(conn, cfg, baseDir, dialect) {
+    const pool = new ParsePool();
+    return {
+        build: (forcedPaths)=>reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),
+        invalidate: (kind)=>kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool),
+        invalidateFeatures: (toggles)=>invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),
+        close: ()=>pool.close(),
+        get poolsCreated () {
+            return pool.poolsCreated;
+        }
+    };
+}

package/dist/esm/store/builder.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["embed","FEATURES","rank","listFiles","ParsePool","reparseFiles","NARROW_FEATURE_TABLE","reconcile","getColumns","withTransaction","nullEmbedVectors","conn","dialect","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","prepare","existingPaths","Set","all","map","r","path","files","filter","f","has","relPath","length","docs","undefined","paths","d","delta","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","name","feature","find","Error","turnedOn","docsForFeature","afterReconcile","nullRank","columns","rerunRank","invalidateFeatureToggles","toggles","byName","Map","linksToggle","get","rankToggle","createBuilder","build","forcedPaths","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":"AACA,SAASA,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,IAAI,QAAQ,sBAAsB;AAE3C,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,YAAY,QAAQ,qBAAqB;AAGlD,SAASC,oBAAoB,QAAQ,qBAAqB;AAC1D,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,eAAe,QAAQ,mBAAmB;AAsBnD,6FAA6F;AAC7F,wEAAwE;AACxE,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;IACzE,MAAMH,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,sDAAsDD,QAAQE,SAAS;IACnH,OAAO;QAAEC,QAAQ;IAAE;AACrB;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,eAAeC,kBAAkBL,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe;IACzH,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,gBAAgB,IAAIC,IAAI,AAAC,CAAA,MAAMH,aAAaI,GAAG,EAAC,EAAGC,GAAG,CAAC,CAACC,IAAM,AAACA,EAAuBC,IAAI;IAChG,8FAA8F;IAC9F,4FAA4F;IAC5F,MAAMC,QAAQzB,UAAUc,KAAKC,SAASW,MAAM,CAAC,CAACC,IAAMA,EAAE9B,KAAK,IAAIsB,cAAcS,GAAG,CAACD,EAAEE,OAAO;IAC1F,IAAIJ,MAAMK,MAAM,KAAK,GAAG,OAAO;QAAElB,QAAQ;IAAE;IAE3C,MAAM,EAAEmB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAAC5B;KAAM,EAAEiB,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACtF,MAAMiB,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IAChF,MAAMhC,gBACJE,MACA;YACQX,eACAA;QADN,QAAMA,gBAAAA,MAAM0C,MAAM,cAAZ1C,oCAAAA,mBAAAA,OAAeW,MAAMyB,OAAOE;QAClC,QAAMtC,eAAAA,MAAM2C,KAAK,cAAX3C,mCAAAA,kBAAAA,OACJW,MACAuB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;gBAAEV,MAAMU,EAAEL,OAAO;gBAAEY,WAAWP,EAAEO,SAAS,CAAC5C,KAAK;YAAC,CAAA,IACjEsC;IAEJ,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,eAAeY,uBAAuBlC,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAE2B,MAAqB;IACrJ,MAAMC,QAAQzC,oBAAoB,CAACwC,OAAOE,IAAI,CAAC;IAC/C,MAAMC,UAAUhD,SAASiD,IAAI,CAAC,CAACpB,IAAMA,EAAEkB,IAAI,KAAKF,OAAOE,IAAI;IAC3D,IAAI,CAACC,SAAS,MAAM,IAAIE,MAAM,CAAC,qDAAqD,EAAEL,OAAOE,IAAI,CAAC,CAAC,CAAC;IAEpG,IAAI,CAACF,OAAOM,QAAQ,EAAE;QACpB,MAAM3C,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO,GAAGnC,QAAQE,SAAS;QACtF,OAAO;YAAEC,QAAQ;QAAE;IACrB;IAEA,MAAMa,QAAQzB,UAAUc,KAAKC;IAC7B,MAAM,EAAEgB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAACqB;KAAQ,EAAEhC,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACxF,MAAMkC,iBAAiBnB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;YAAEV,MAAMU,EAAEL,OAAO;YAAEY,WAAWP,EAAEO,SAAS,CAACK,QAAQD,IAAI,CAAC;QAAC,CAAA;IAChG,MAAMZ,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,gGAAgG;IAChG,8FAA8F;IAC9F,8CAA8C;IAC9C,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAOJ;QAAOK,UAAU,EAAE;IAAC;IACnF,MAAMhC,gBACJE,MACA;YAEQsC,gBACAA;QAFN,MAAMtC,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO;QACtC,QAAME,iBAAAA,QAAQN,KAAK,cAAbM,qCAAAA,oBAAAA,SAAgBtC,MAAM0C,gBAAgBf;QAC5C,QAAMW,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyBtC,MAAM2B;IACvC,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,eAAesB,SAAS5C,IAAgB,EAAEC,OAAyB;IACjE,MAAM4C,UAAU,MAAMhD,WAAWG;IACjC,IAAI,CAAC6C,QAAQzB,GAAG,CAAC,UAAU;IAC3B,MAAMtB,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,0CAA0CD,QAAQE,SAAS;AACzG;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,eAAe2C,UAAU9C,IAAgB,EAAEC,OAAyB;IAClE,MAAM0B,QAAwB;QAAEV,OAAO,EAAE;QAAEW,UAAU,EAAE;QAAEC,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IACjF,MAAMhC,gBACJE,MACA;YACQT;QAAN,QAAMA,uBAAAA,KAAKoD,cAAc,cAAnBpD,2CAAAA,0BAAAA,MAAsBS,MAAM2B;IACpC,GACA1B,QAAQE,SAAS;AAErB;AAEA,eAAe4C,yBAAyB/C,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAEwC,OAAwB;IAC1J,MAAMC,SAAS,IAAIC,IAAIF,QAAQlC,GAAG,CAAC,CAACqB,SAAW;YAACA,OAAOE,IAAI;YAAEF;SAAO;IACpE,IAAI/B,SAAS;IAEb,4FAA4F;IAC5F,mFAAmF;IACnF,MAAM+C,cAAcF,OAAOG,GAAG,CAAC;IAC/B,IAAID,aAAa;QACf/C,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2C,YAAW,EAAG/C,MAAM;QAC/F,4FAA4F;QAC5F,0FAA0F;QAC1F,IAAI,CAAC+C,YAAYV,QAAQ,EAAE,MAAMG,SAAS5C,MAAMC;IAClD;IAEA,wFAAwF;IACxF,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMoD,aAAaJ,OAAOG,GAAG,CAAC;IAC9B,IAAIC,YAAY;QACd,IAAIA,WAAWZ,QAAQ,EAAE,MAAMK,UAAU9C,MAAMC;aAC1C,MAAM2C,SAAS5C,MAAMC;IAC5B;IAEA,KAAK,MAAMkC,UAAUa,QAAS;QAC5B,IAAIb,OAAOE,IAAI,KAAK,WAAWF,OAAOE,IAAI,KAAK,QAAQ;QACvDjC,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2B,OAAM,EAAG/B,MAAM;IAC5F;IACA,OAAO;QAAEA;IAAO;AAClB;AAEA,4FAA4F;AAC5F,6DAA6D;AAC7D,OAAO,SAASkD,cAActD,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB;IACrG,MAAMO,OAAO,IAAIf;IACjB,OAAO;QACL8D,OAAO,CAACC,cAAgB5D,UAAUI,MAAMM,KAAKC,SAASN,SAASO,MAAMgD;QACrEC,YAAY,CAACC,OAAUA,SAAS,UAAU3D,iBAAiBC,MAAMC,WAAWI,kBAAkBL,MAAMM,KAAKC,SAASN,SAASO;QAC3HmD,oBAAoB,CAACX,UAAYD,yBAAyB/C,MAAMM,KAAKC,SAASN,SAASO,MAAMwC;QAC7FY,OAAO,IAAMpD,KAAKoD,KAAK;QACvB,IAAIC;YACF,OAAOrD,KAAKqD,YAAY;QAC1B;IACF;AACF"}

package/dist/esm/store/duckdb/lexical.d.ts

@@ -2,6 +2,7 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
+export declare function markContentStale(conn: Connection): void;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;

package/dist/esm/store/duckdb/lexical.js

@@ -50,6 +50,24 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
+// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
+const ftsState = new WeakMap();
+function stateFor(conn) {
+    let state = ftsState.get(conn);
+    if (!state) {
+        state = {
+            stale: true
+        };
+        ftsState.set(conn, state);
+    }
+    return state;
+}
+// Called whenever this store's reconcileContent changes `content`, so the next lexical query
+// rebuilds instead of searching stale content.
+export function markContentStale(conn) {
+    stateFor(conn).stale = true;
+}
 // A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
 // (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
 function splitTerms(terms) {
@@ -142,16 +160,11 @@ export async function queryLexical(conn, terms, opts, state) {
     const stmt = await conn.prepare(sql);
     return await stmt.all(...params, limit);
 }
-// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),
-// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.
+// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches
+// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.
 export function createLexicalIndex(conn) {
-    const state = {
-        stale: true
-    };
     return {
-        query: (terms, opts)=>queryLexical(conn, terms, opts, state),
-        markStale: ()=>{
-            state.stale = true;
-        }
+        query: (terms, opts)=>queryLexical(conn, terms, opts, stateFor(conn)),
+        markStale: ()=>markContentStale(conn)
     };
 }

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),\n// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","hasUnspacedRun","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACvB,eAAeuB,SAAS;YAAA,KAAK,MAAMG,KAAKH,OAAOI,KAAK,CAAC,OAAQ,IAAID,EAAEF,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACC;QAAE;QACjG,OAAO;IACT;IACA,KAAK,MAAME,OAAOhB,eAAee,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACtB,IAAIxB,eAAe4B,MAAMR,WAAWK,IAAI,CAACG;aACpCT,MAAMM,IAAI,CAACG;IAClB;IACA,OAAO;QAAET;QAAOC;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,eAAeS,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMhC,gBAAgB6B,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASE,kBAAkBf,KAAe,EAAEC,UAAoB;IAC9D,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAInB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMe,YAAYpB,MAAMqB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAASvC,OAAQ;YAC1BiC,WAAWV,IAAI,CAAC,GAAGtB,YAAY,CAACsC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYZ,IAAI,CAACc;QACnB;QACAH,UAAUX,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGa,WAAWb,IAAI,CAACc;IAClB;IAEA,KAAK,MAAMG,OAAOtB,WAAY;QAC5B,MAAMuB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAASvC,OAAQ;YAC1BiC,WAAWV,IAAI,CAAC,GAAGtB,YAAY,CAACsC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYZ,IAAI,CAACkB;QACnB;QACAP,UAAUX,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHa,WAAWb,IAAI,CAACkB;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,kHAAkH;AAClH,yIAAyI;AACzI,OAAO,eAAeU,aAAalB,IAAgB,EAAEnB,KAAa,EAAEsC,IAAyB,EAAElB,KAAoB;IACjH,MAAMmB,cAAcxC,oBAAoBC;IACxC,IAAIuC,gBAAgB,MAAM;QACxB,MAAM,IAAInD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEmD,YAAY1C,KAAK,CAAC,GAAG,EAAE0C,YAAYlC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEkC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE9B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMK,eAAeC,MAAMC;IAEjD,MAAM,EAAEc,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBf,OAAOC;IAC/D,MAAMmC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAM1B,KAAK2B,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,+HAA+H;AAC/H,qHAAqH;AACrH,OAAO,SAASK,mBAAmB7B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,CAACjD,OAAOsC,OAASD,aAAalB,MAAMnB,OAAOsC,MAAMlB;QACxD8B,WAAW;YACT9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n  let state = ftsState.get(conn);\n  if (!state) {\n    state = { stale: true };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n  stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","hasUnspacedRun","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","markContentStale","splitTerms","words","substrings","_m","inner","phrase","length","push","w","split","tok","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,SAASI,iBAAiBL,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASG,WAAWhB,KAAa;IAC/B,MAAMiB,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMjB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACiB,IAAIC;QACtD,MAAMC,SAASD,MAAMd,IAAI;QACzB,IAAIe,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAChC,eAAegC,SAAS;YAAA,KAAK,MAAMG,KAAKH,OAAOI,KAAK,CAAC,OAAQ,IAAID,EAAEF,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACC;QAAE;QACjG,OAAO;IACT;IACA,KAAK,MAAME,OAAOzB,eAAewB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACtB,IAAIjC,eAAeqC,MAAMR,WAAWK,IAAI,CAACG;aACpCT,MAAMM,IAAI,CAACG;IAClB;IACA,OAAO;QAAET;QAAOC;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,eAAeS,eAAejB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClB,MAAMH,KAAKkB,IAAI,CAAC;IAChB,MAAMtC,gBAAgBoB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKkB,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAjB,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASgB,kBAAkBZ,KAAe,EAAEC,UAAoB;IAC9D,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIhB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYjB,MAAMkB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAAS7C,OAAQ;YAC1BuC,WAAWP,IAAI,CAAC,GAAG/B,YAAY,CAAC4C,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYT,IAAI,CAACW;QACnB;QACAH,UAAUR,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGU,WAAWV,IAAI,CAACW;IAClB;IAEA,KAAK,MAAMG,OAAOnB,WAAY;QAC5B,MAAMoB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAAS7C,OAAQ;YAC1BuC,WAAWP,IAAI,CAAC,GAAG/B,YAAY,CAAC4C,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYT,IAAI,CAACe;QACnB;QACAP,UAAUR,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHU,WAAWV,IAAI,CAACe;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,kHAAkH;AAClH,yIAAyI;AACzI,OAAO,eAAeU,aAAajC,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;IACjH,MAAMkC,cAAc9C,oBAAoBC;IACxC,IAAI6C,gBAAgB,MAAM;QACxB,MAAM,IAAIzD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEyD,YAAYhD,KAAK,CAAC,GAAG,EAAEgD,YAAYxC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEwC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE3B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWhB;IACzC,IAAIiB,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMK,eAAejB,MAAMC;IAEjD,MAAM,EAAE6B,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBZ,OAAOC;IAC/D,MAAMgC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAMzC,KAAK0C,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,2FAA2F;AAC3F,gGAAgG;AAChG,OAAO,SAASK,mBAAmB5C,IAAgB;IACjD,OAAO;QACL6C,OAAO,CAACvD,OAAO4C,OAASD,aAAajC,MAAMV,OAAO4C,MAAMnC,SAASC;QACjE8C,WAAW,IAAMzC,iBAAiBL;IACpC;AACF"}

package/dist/esm/store/duckdb/open.d.ts

@@ -1,12 +1,13 @@
+import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "2";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
+export declare const SCHEMA_VERSION = "3";
+export type { OpenResult };
+interface DuckdbHandle {
+    instance: DuckDBInstance;
+    duckdb: DuckDBConnection;
 }
+export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
 export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;