sensemaking 0.19.1 → 0.20.0

package/dist/esm/chunk/index.d.ts

@@ -1,7 +1,7 @@
 import type { Block, Chunk, ChunkOptions } from './types.js';
 export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
 export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
-export { extractText } from './extract.js';
+export { extractText, extractTexts } from './extract.js';
 export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
 export { parse } from './parse.js';
 export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';

package/dist/esm/chunk/index.js

@@ -10,7 +10,7 @@ export function chunkFromBlocks(blocks, body, opts) {
 export function chunk(body, opts) {
     return chunkFromBlocks(parse(body), body, opts);
 }
-export { extractText } from './extract.js';
+export { extractText, extractTexts } from './extract.js';
 export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
 export { parse } from './parse.js';
 export { CHUNK_VERSION } from './version.js';

package/dist/esm/chunk/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["group","parse","chunkFromBlocks","blocks","body","opts","chunk","extractText","DEFAULT_TARGET_TOKENS","estimateTokens","CHUNK_VERSION"],"mappings":"AAAA,SAASA,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AAGnC,4FAA4F;AAC5F,qEAAqE;AACrE,OAAO,SAASC,gBAAgBC,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOL,MAAMG,QAAQC,MAAMC;AAC7B;AAEA,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,SAASC,MAAMF,IAAY,EAAEC,IAAmB;IACrD,OAAOH,gBAAgBD,MAAMG,OAAOA,MAAMC;AAC5C;AAEA,SAASE,WAAW,QAAQ,eAAe;AAC3C,SAASC,qBAAqB,EAAEC,cAAc,QAAQ,aAAa;AACnE,SAASR,KAAK,QAAQ,aAAa;AAEnC,SAASS,aAAa,QAAQ,eAAe"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText, extractTexts } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["group","parse","chunkFromBlocks","blocks","body","opts","chunk","extractText","extractTexts","DEFAULT_TARGET_TOKENS","estimateTokens","CHUNK_VERSION"],"mappings":"AAAA,SAASA,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AAGnC,4FAA4F;AAC5F,qEAAqE;AACrE,OAAO,SAASC,gBAAgBC,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOL,MAAMG,QAAQC,MAAMC;AAC7B;AAEA,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,SAASC,MAAMF,IAAY,EAAEC,IAAmB;IACrD,OAAOH,gBAAgBD,MAAMG,OAAOA,MAAMC;AAC5C;AAEA,SAASE,WAAW,EAAEC,YAAY,QAAQ,eAAe;AACzD,SAASC,qBAAqB,EAAEC,cAAc,QAAQ,aAAa;AACnE,SAAST,KAAK,QAAQ,aAAa;AAEnC,SAASU,aAAa,QAAQ,eAAe"}

package/dist/esm/index.d.ts

@@ -7,7 +7,7 @@ export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
 export type { OpenResult } from './store/index.js';
-export { clearCache, openStore as open } from './store/index.js';
+export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js

@@ -4,5 +4,5 @@ export { mapTree, peek, presetCoverage, search } from './commands/index.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
 export { SenseError } from './errors.js';
 export { printRows } from './output/output.js';
-export { clearCache, openStore as open } from './store/index.js';
+export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAIjE,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","CAPABILITY_NAMES","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,gBAAgB,EAAEC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAInF,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/scan/pool.d.ts

@@ -0,0 +1,15 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from './list.js';
+export type FileResult = {
+    doc: ParsedDoc;
+    warnings: string[];
+};
+export declare class ParsePool {
+    private pool;
+    poolsCreated: number;
+    private ensure;
+    run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]>;
+    close(): Promise<void>;
+}

package/dist/esm/scan/pool.js

@@ -0,0 +1,77 @@
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { reviveError } from './worker-error.js';
+// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
+const _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;
+// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
+// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
+let workerFile;
+function resolveWorkerFile() {
+    if (workerFile) return workerFile;
+    const load = createRequire(import.meta.url);
+    for (const rel of [
+        '..',
+        '../..',
+        '../../..'
+    ]){
+        try {
+            if (load(`${rel}/package.json`).name === 'sensemaking') {
+                workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');
+                return workerFile;
+            }
+        } catch  {}
+    }
+    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
+}
+// One Tinypool per instance, created lazily on first dispatch and reused by every later call:
+// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.
+export class ParsePool {
+    ensure(workerData, maxWorkers) {
+        if (!this.pool) {
+            const { Tinypool } = _require('tinypool');
+            this.pool = new Tinypool({
+                filename: resolveWorkerFile(),
+                minThreads: maxWorkers,
+                maxThreads: maxWorkers,
+                workerData
+            });
+            this.poolsCreated++;
+        }
+        return this.pool;
+    }
+    // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
+    // extracted text and feature values, never the mdast tree.
+    async run(files, features, cfg, onParsed, maxWorkers) {
+        // cfg and the feature selection are constant for the dispatch, so they cross once per worker
+        // as workerData. Features carry closures and cannot cross at all; only their names do.
+        const workerData = {
+            cfg,
+            featureNames: features.map((feature)=>feature.name)
+        };
+        const pool = this.ensure(workerData, maxWorkers);
+        let done = 0;
+        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
+        // regardless of task completion order, which first-seen column order depends on downstream.
+        return Promise.all(files.map(async (file)=>{
+            const result = await pool.run(file);
+            if (!result.ok) throw reviveError(result.error);
+            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+            return {
+                doc: result.doc,
+                warnings: result.warnings
+            };
+        }));
+    }
+    // A pool never created costs nothing to destroy.
+    async close() {
+        if (!this.pool) return;
+        const pool = this.pool;
+        this.pool = undefined;
+        await pool.destroy();
+    }
+    constructor(){
+        // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;
+        // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.
+        this.poolsCreated = 0;
+    }
+}

package/dist/esm/scan/pool.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\nexport type FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["createRequire","dirname","join","reviveError","_require","require","url","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","ParsePool","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","featureNames","map","feature","done","Promise","all","file","result","ok","error","doc","warnings","close","undefined","destroy"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAS1C,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcL,cAAc,YAAYM,GAAG,IAAID;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOT,cAAc,YAAYM,GAAG;IAC1C,KAAK,MAAMI,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaL,KAAKD,QAAQQ,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC;IAMHC,OAAOC,UAA2B,EAAEC,UAAkB,EAAuB;QACnF,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,MAAM,EAAEC,QAAQ,EAAE,GAAGf,SAAS;YAC9B,IAAI,CAACc,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUZ;gBAAqBa,YAAYJ;gBAAYK,YAAYL;gBAAYD;YAAW;YACrH,IAAI,CAACO,YAAY;QACnB;QACA,OAAO,IAAI,CAACL,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,MAAMM,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEX,UAAkB,EAAyB;QACxJ,6FAA6F;QAC7F,uFAAuF;QACvF,MAAMD,aAA8B;YAAEW;YAAKE,cAAcH,SAASI,GAAG,CAAC,CAACC,UAAYA,QAAQpB,IAAI;QAAE;QACjG,MAAMO,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;QACrC,IAAIe,OAAO;QACX,0FAA0F;QAC1F,4FAA4F;QAC5F,OAAOC,QAAQC,GAAG,CAChBT,MAAMK,GAAG,CAAC,OAAOK;YACf,MAAMC,SAAU,MAAMlB,KAAKM,GAAG,CAACW;YAC/B,IAAI,CAACC,OAAOC,EAAE,EAAE,MAAMlC,YAAYiC,OAAOE,KAAK;YAC9CV,qBAAAA,+BAAAA,SAAW,EAAEI;YACb,OAAO;gBAAEO,KAAKH,OAAOG,GAAG;gBAAEC,UAAUJ,OAAOI,QAAQ;YAAC;QACtD;IAEJ;IAEA,iDAAiD;IACjD,MAAMC,QAAuB;QAC3B,IAAI,CAAC,IAAI,CAACvB,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAGwB;QACZ,MAAMxB,KAAKyB,OAAO;IACpB;;QAvCA,2FAA2F;QAC3F,uGAAuG;aACvGpB,eAAe;;AAsCjB"}

package/dist/esm/scan/reparse.d.ts

@@ -2,6 +2,7 @@ import type { Config } from '../config/index.js';
 import type { Feature } from '../features/types.js';
 import type { ParsedDoc } from './index.js';
 import type { FileStat } from './list.js';
+import { ParsePool } from './pool.js';
 export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
@@ -10,6 +11,7 @@ export interface ReparseResult {
 export interface ReparseOptions {
     threshold?: number;
     maxWorkers?: number;
+    pool?: ParsePool;
 }
 export declare function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[];
 export declare function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options?: ReparseOptions): Promise<ReparseResult>;

package/dist/esm/scan/reparse.js

@@ -1,35 +1,10 @@
-import { createRequire } from 'node:module';
 import { availableParallelism } from 'node:os';
-import { dirname, join } from 'node:path';
 import { parseFile } from './index.js';
-import { reviveError } from './worker-error.js';
-// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
-// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.
-const _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;
+import { ParsePool } from './pool.js';
 // Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files
 // (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.
 const DEFAULT_THRESHOLD = 200;
 const DEFAULT_MAX_WORKERS = 8;
-// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
-// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
-let workerFile;
-function resolveWorkerFile() {
-    if (workerFile) return workerFile;
-    const load = createRequire(import.meta.url);
-    for (const rel of [
-        '..',
-        '../..',
-        '../../..'
-    ]){
-        try {
-            if (load(`${rel}/package.json`).name === 'sensemaking') {
-                workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');
-                return workerFile;
-            }
-        } catch  {}
-    }
-    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
-}
 // Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and
 // the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).
 export function featuresForFile(features, cfg, file) {
@@ -45,41 +20,19 @@ function reparseSerial(files, features, cfg, onParsed) {
     }
     return results;
 }
-// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
-// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.
-async function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
-    const { Tinypool } = _require('tinypool');
-    // cfg and the feature selection are constant for the dispatch, so they cross once per worker
-    // as workerData. Features carry closures and cannot cross at all; only their names do.
-    const workerData = {
-        cfg,
-        featureNames: features.map((feature)=>feature.name)
-    };
-    const pool = new Tinypool({
-        filename: resolveWorkerFile(),
-        minThreads: maxWorkers,
-        maxThreads: maxWorkers,
-        workerData
-    });
-    let done = 0;
+// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with
+// none given, opens an ephemeral ParsePool for this call alone and always closes it.
+async function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
+    if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);
+    const ephemeral = new ParsePool();
     try {
-        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
-        // regardless of task completion order, which first-seen column order depends on downstream.
-        const results = await Promise.all(files.map(async (file)=>{
-            const result = await pool.run(file);
-            if (!result.ok) throw reviveError(result.error);
-            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
-            return {
-                doc: result.doc,
-                warnings: result.warnings
-            };
-        }));
-        await pool.destroy();
+        const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);
+        await ephemeral.close();
         return results;
     } catch (err) {
-        // Tear the pool down before rethrowing, but a destroy failure must not mask the parse
+        // Tear the pool down before rethrowing, but a close failure must not mask the parse
         // failure that caused it -- not a `finally`, for that reason.
-        await pool.destroy().catch(()=>{});
+        await ephemeral.close().catch(()=>{});
         throw err;
     }
 }
@@ -89,7 +42,7 @@ export async function reparseFiles(files, features, cfg, knownColumns, onParsed,
     var _options_threshold, _options_maxWorkers;
     const threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : DEFAULT_THRESHOLD;
     const maxWorkers = (_options_maxWorkers = options.maxWorkers) !== null && _options_maxWorkers !== void 0 ? _options_maxWorkers : Math.min(DEFAULT_MAX_WORKERS, availableParallelism());
-    const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);
+    const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);
     const docs = [];
     const warnings = [];
     const newColumns = [];

package/dist/esm/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\n// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n  // as workerData. Features carry closures and cannot cross at all; only their names do.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["createRequire","availableParallelism","dirname","join","parseFile","reviveError","_require","require","url","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","Tinypool","workerData","featureNames","map","pool","filename","minThreads","maxThreads","Promise","all","result","run","ok","error","doc","warnings","destroy","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","newColumns","seen","Set","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,oBAAoB,QAAQ,UAAU;AAC/C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAQ1C,SAASC,SAAS,QAAQ,aAAa;AAEvC,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,qIAAqI;AACrI,MAAMC,WAAW,OAAOC,YAAY,cAAcP,cAAc,YAAYQ,GAAG,IAAID;AAmBnF,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAME,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOb,cAAc,YAAYQ,GAAG;IAC1C,KAAK,MAAMM,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaR,KAAKD,QAAQW,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAAC3B,UAAUiB,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB;IAClJ,MAAM,EAAEC,QAAQ,EAAE,GAAG5B,SAAS;IAC9B,6FAA6F;IAC7F,uFAAuF;IACvF,MAAM6B,aAA8B;QAAEf;QAAKgB,cAAcjB,SAASkB,GAAG,CAAC,CAACd,UAAYA,QAAQR,IAAI;IAAE;IACjG,MAAMuB,OAAO,IAAIJ,SAAS;QAAEK,UAAU3B;QAAqB4B,YAAYP;QAAYQ,YAAYR;QAAYE;IAAW;IACtH,IAAIN,OAAO;IACX,IAAI;QACF,0FAA0F;QAC1F,4FAA4F;QAC5F,MAAMD,UAAU,MAAMc,QAAQC,GAAG,CAC/BjB,MAAMW,GAAG,CAAC,OAAOhB;YACf,MAAMuB,SAAU,MAAMN,KAAKO,GAAG,CAACxB;YAC/B,IAAI,CAACuB,OAAOE,EAAE,EAAE,MAAMzC,YAAYuC,OAAOG,KAAK;YAC9CpB,qBAAAA,+BAAAA,SAAW,EAAEE;YACb,OAAO;gBAAEmB,KAAKJ,OAAOI,GAAG;gBAAEC,UAAUL,OAAOK,QAAQ;YAAC;QACtD;QAEF,MAAMX,KAAKY,OAAO;QAClB,OAAOtB;IACT,EAAE,OAAOuB,KAAK;QACZ,sFAAsF;QACtF,8DAA8D;QAC9D,MAAMb,KAAKY,OAAO,GAAGE,KAAK,CAAC,KAAO;QAClC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAa3B,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEkC,YAAiC,EAAE3B,QAAiC,EAAE4B,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB9C;IACvC,MAAMwB,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAChD,qBAAqBT;IACvE,MAAM2B,UAAUF,MAAMiC,MAAM,IAAIH,YAAY,MAAMxB,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,cAAcR,cAAcC,OAAOP,UAAUC,KAAKO;IAElJ,MAAMiC,OAAoB,EAAE;IAC5B,MAAMX,WAAqB,EAAE;IAC7B,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIT;IACrB,KAAK,MAAM,EAAEN,GAAG,EAAEC,UAAUe,YAAY,EAAE,IAAIpC,QAAS;QACrDqB,SAASlB,IAAI,IAAIiC;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACnB,IAAIoB,IAAI,EAAG;YACvC,IAAI,CAACN,KAAKO,GAAG,CAACJ,MAAM;gBAClBH,KAAKQ,GAAG,CAACL;gBACTJ,WAAW9B,IAAI,CAACkC;YAClB;QACF;QACAL,KAAK7B,IAAI,CAACiB;IACZ;IAEA,OAAO;QAAEY;QAAMX;QAAUY;IAAW;AACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["availableParallelism","parseFile","ParsePool","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","pool","run","ephemeral","close","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","warnings","newColumns","seen","Set","doc","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,oBAAoB,QAAQ,UAAU;AAI/C,SAASC,SAAS,QAAQ,aAAa;AAGvC,SAASC,SAAS,QAAQ,YAAY;AAsBtC,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAMC,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAACjB,UAAUO,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB,EAAEC,IAA2B;IAC/K,IAAIA,MAAM,OAAOA,KAAKC,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;IAC1D,MAAMG,YAAY,IAAIrB;IACtB,IAAI;QACF,MAAMa,UAAU,MAAMQ,UAAUD,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;QACpE,MAAMG,UAAUC,KAAK;QACrB,OAAOT;IACT,EAAE,OAAOU,KAAK;QACZ,oFAAoF;QACpF,8DAA8D;QAC9D,MAAMF,UAAUC,KAAK,GAAGE,KAAK,CAAC,KAAO;QACrC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAad,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEqB,YAAiC,EAAEd,QAAiC,EAAEe,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;IACvC,MAAMiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAC5B,qBAAqBJ;IACvE,MAAMe,UAAUF,MAAMoB,MAAM,IAAIH,YAAY,MAAMX,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,YAAYS,QAAQR,IAAI,IAAIT,cAAcC,OAAOP,UAAUC,KAAKO;IAEhK,MAAMoB,OAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIV;IACrB,KAAK,MAAM,EAAEW,GAAG,EAAEJ,UAAUK,YAAY,EAAE,IAAIzB,QAAS;QACrDoB,SAASjB,IAAI,IAAIsB;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACP,KAAKQ,GAAG,CAACJ,MAAM;gBAClBJ,KAAKS,GAAG,CAACL;gBACTL,WAAWlB,IAAI,CAACuB;YAClB;QACF;QACAP,KAAKhB,IAAI,CAACqB;IACZ;IAEA,OAAO;QAAEL;QAAMC;QAAUC;IAAW;AACtC"}

package/dist/esm/store/builder.d.ts

@@ -0,0 +1,11 @@
+import type { Config } from '../config/index.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export interface Builder {
+    build(): Promise<{
+        parsed: number;
+        warnings: string[];
+    }>;
+    close(): Promise<void>;
+    readonly poolsCreated: number;
+}
+export declare function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder;

package/dist/esm/store/builder.js

@@ -0,0 +1,14 @@
+import { ParsePool } from '../scan/pool.js';
+import { reconcile } from './reconcile.js';
+// The pool is created at most once, lazily, on whichever build() call first needs it, and reused
+// by every later build() on this instance.
+export function createBuilder(conn, cfg, baseDir, dialect) {
+    const pool = new ParsePool();
+    return {
+        build: ()=>reconcile(conn, cfg, baseDir, dialect, pool),
+        close: ()=>pool.close(),
+        get poolsCreated () {
+            return pool.poolsCreated;
+        }
+    };
+}

package/dist/esm/store/builder.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reconcile } from './reconcile.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  build(): Promise<{ parsed: number; warnings: string[] }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// The pool is created at most once, lazily, on whichever build() call first needs it, and reused\n// by every later build() on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: () => reconcile(conn, cfg, baseDir, dialect, pool),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["ParsePool","reconcile","createBuilder","conn","cfg","baseDir","dialect","pool","build","close","poolsCreated"],"mappings":"AACA,SAASA,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,SAAS,QAAQ,iBAAiB;AAa3C,iGAAiG;AACjG,2CAA2C;AAC3C,OAAO,SAASC,cAAcC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB;IACrG,MAAMC,OAAO,IAAIP;IACjB,OAAO;QACLQ,OAAO,IAAMP,UAAUE,MAAMC,KAAKC,SAASC,SAASC;QACpDE,OAAO,IAAMF,KAAKE,KAAK;QACvB,IAAIC;YACF,OAAOH,KAAKG,YAAY;QAC1B;IACF;AACF"}

package/dist/esm/store/duckdb/fieldStats.js

@@ -7,25 +7,38 @@ function classifyVariantType(name, field) {
     if (name === 'VARCHAR') return 'text';
     throw new Error(`duckdb: variant_typeof() returned unrecognized type "${name}" for column "${field}"`);
 }
-// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
+// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in
+// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same
+// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:
+// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking
+// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no
+// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300
+// columns runs in 7ms.
+const AGGREGATES_PER_QUERY = 16;
+// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
 // '|' is the separator, not ',': DECIMAL's display name ("DECIMAL(4, 2)") contains a comma.
 export async function fieldStats(conn, columns, scopeWhere) {
-    const exprs = columns.map((name, i)=>{
-        const quoted = quoteIdent(name);
-        return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
-    });
-    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
-    const row = await stmt.get();
-    return columns.map((name, i)=>{
-        var _row_, _row_1;
-        const raw = ((_row_ = row[`t${i}`]) !== null && _row_ !== void 0 ? _row_ : '').split('|').filter(Boolean);
-        const types = [
-            ...new Set(raw.map((t)=>classifyVariantType(t, name)))
-        ].sort();
-        return {
-            field: name,
-            coverage: Number((_row_1 = row[`n${i}`]) !== null && _row_1 !== void 0 ? _row_1 : 0),
-            type: types.join(',')
-        };
-    });
+    const stats = [];
+    for(let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY){
+        const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);
+        const exprs = chunk.map((name, i)=>{
+            const quoted = quoteIdent(name);
+            return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
+        });
+        const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
+        const row = await stmt.get();
+        for (const [i, name] of chunk.entries()){
+            var _row_, _row_1;
+            const raw = ((_row_ = row[`t${i}`]) !== null && _row_ !== void 0 ? _row_ : '').split('|').filter(Boolean);
+            const types = [
+                ...new Set(raw.map((t)=>classifyVariantType(t, name)))
+            ].sort();
+            stats.push({
+                field: name,
+                coverage: Number((_row_1 = row[`n${i}`]) !== null && _row_1 !== void 0 ? _row_1 : 0),
+                type: types.join(',')
+            });
+        }
+    }
+    return stats;
 }

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => {\n    const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n    const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n    return { field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') };\n  });\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","fieldStats","conn","columns","scopeWhere","exprs","map","i","quoted","stmt","prepare","join","row","get","raw","split","filter","Boolean","types","Set","t","sort","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,8IAA8I;AAC9I,4FAA4F;AAC5F,OAAO,eAAeI,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACV,MAAMW;QAC/B,MAAMC,SAASd,WAAWE;QAC1B,OAAO,CAAC,MAAM,EAAEY,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IACvI;IACA,MAAME,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEP,YAAY;IAC3F,MAAMQ,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOV,QAAQG,GAAG,CAAC,CAACV,MAAMW;YACVK,OAEyBA;QAFvC,MAAME,MAAM,EAAEF,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAIG,KAAK,CAAC,KAAKC,MAAM,CAACC;QAC/D,MAAMC,QAAQ;eAAI,IAAIC,IAAIL,IAAIR,GAAG,CAAC,CAACc,IAAMzB,oBAAoByB,GAAGxB;SAAQ,CAACyB,IAAI;QAC7E,OAAO;YAAExB,OAAOD;YAAM0B,UAAUC,QAAOX,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;YAAIY,MAAMN,MAAMP,IAAI,CAAC;QAAK;IACnF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in\n// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same\n// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:\n// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking\n// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no\n// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300\n// columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,qGAAqG;AACrG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,uBAAuB;AACvB,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}

package/dist/esm/store/duckdb/lexical.d.ts

@@ -2,6 +2,7 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
+export declare function markContentStale(conn: Connection): void;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;

package/dist/esm/store/duckdb/lexical.js

@@ -50,6 +50,24 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
+// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
+const ftsState = new WeakMap();
+function stateFor(conn) {
+    let state = ftsState.get(conn);
+    if (!state) {
+        state = {
+            stale: true
+        };
+        ftsState.set(conn, state);
+    }
+    return state;
+}
+// Called whenever this store's reconcileContent changes `content`, so the next lexical query
+// rebuilds instead of searching stale content.
+export function markContentStale(conn) {
+    stateFor(conn).stale = true;
+}
 // A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
 // (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
 function splitTerms(terms) {
@@ -142,16 +160,11 @@ export async function queryLexical(conn, terms, opts, state) {
     const stmt = await conn.prepare(sql);
     return await stmt.all(...params, limit);
 }
-// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),
-// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.
+// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches
+// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.
 export function createLexicalIndex(conn) {
-    const state = {
-        stale: true
-    };
     return {
-        query: (terms, opts)=>queryLexical(conn, terms, opts, state),
-        markStale: ()=>{
-            state.stale = true;
-        }
+        query: (terms, opts)=>queryLexical(conn, terms, opts, stateFor(conn)),
+        markStale: ()=>markContentStale(conn)
     };
 }

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),\n// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","hasUnspacedRun","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACvB,eAAeuB,SAAS;YAAA,KAAK,MAAMG,KAAKH,OAAOI,KAAK,CAAC,OAAQ,IAAID,EAAEF,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACC;QAAE;QACjG,OAAO;IACT;IACA,KAAK,MAAME,OAAOhB,eAAee,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACtB,IAAIxB,eAAe4B,MAAMR,WAAWK,IAAI,CAACG;aACpCT,MAAMM,IAAI,CAACG;IAClB;IACA,OAAO;QAAET;QAAOC;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,eAAeS,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMhC,gBAAgB6B,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASE,kBAAkBf,KAAe,EAAEC,UAAoB;IAC9D,MAAMe,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAInB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMe,YAAYpB,MAAMqB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAASvC,OAAQ;YAC1BiC,WAAWV,IAAI,CAAC,GAAGtB,YAAY,CAACsC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYZ,IAAI,CAACc;QACnB;QACAH,UAAUX,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGa,WAAWb,IAAI,CAACc;IAClB;IAEA,KAAK,MAAMG,OAAOtB,WAAY;QAC5B,MAAMuB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAASvC,OAAQ;YAC1BiC,WAAWV,IAAI,CAAC,GAAGtB,YAAY,CAACsC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYZ,IAAI,CAACkB;QACnB;QACAP,UAAUX,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHa,WAAWb,IAAI,CAACkB;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,kHAAkH;AAClH,yIAAyI;AACzI,OAAO,eAAeU,aAAalB,IAAgB,EAAEnB,KAAa,EAAEsC,IAAyB,EAAElB,KAAoB;IACjH,MAAMmB,cAAcxC,oBAAoBC;IACxC,IAAIuC,gBAAgB,MAAM;QACxB,MAAM,IAAInD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEmD,YAAY1C,KAAK,CAAC,GAAG,EAAE0C,YAAYlC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEkC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE9B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMK,eAAeC,MAAMC;IAEjD,MAAM,EAAEc,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBf,OAAOC;IAC/D,MAAMmC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAM1B,KAAK2B,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,+HAA+H;AAC/H,qHAAqH;AACrH,OAAO,SAASK,mBAAmB7B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,CAACjD,OAAOsC,OAASD,aAAalB,MAAMnB,OAAOsC,MAAMlB;QACxD8B,WAAW;YACT9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n  let state = ftsState.get(conn);\n  if (!state) {\n    state = { stale: true };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n  stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["SenseError","hasUnspacedRun","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","markContentStale","splitTerms","words","substrings","_m","inner","phrase","length","push","w","split","tok","ensureFtsFresh","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,MAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAEA,6FAA6F;AAC7F,+CAA+C;AAC/C,OAAO,SAASI,iBAAiBL,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASG,WAAWhB,KAAa;IAC/B,MAAMiB,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMjB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACiB,IAAIC;QACtD,MAAMC,SAASD,MAAMd,IAAI;QACzB,IAAIe,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAChC,eAAegC,SAAS;YAAA,KAAK,MAAMG,KAAKH,OAAOI,KAAK,CAAC,OAAQ,IAAID,EAAEF,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACC;QAAE;QACjG,OAAO;IACT;IACA,KAAK,MAAME,OAAOzB,eAAewB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACtB,IAAIjC,eAAeqC,MAAMR,WAAWK,IAAI,CAACG;aACpCT,MAAMM,IAAI,CAACG;IAClB;IACA,OAAO;QAAET;QAAOC;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,eAAeS,eAAejB,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAME,KAAK,EAAE;IAClB,MAAMH,KAAKkB,IAAI,CAAC;IAChB,MAAMtC,gBAAgBoB,MAAM;QAC1B,6HAA6H;QAC7H,0JAA0J;QAC1J,MAAMA,KAAKkB,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAjB,MAAME,KAAK,GAAG;AAChB;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASgB,kBAAkBZ,KAAe,EAAEC,UAAoB;IAC9D,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIhB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMY,YAAYjB,MAAMkB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAAS7C,OAAQ;YAC1BuC,WAAWP,IAAI,CAAC,GAAG/B,YAAY,CAAC4C,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYT,IAAI,CAACW;QACnB;QACAH,UAAUR,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGU,WAAWV,IAAI,CAACW;IAClB;IAEA,KAAK,MAAMG,OAAOnB,WAAY;QAC5B,MAAMoB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAAS7C,OAAQ;YAC1BuC,WAAWP,IAAI,CAAC,GAAG/B,YAAY,CAAC4C,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYT,IAAI,CAACe;QACnB;QACAP,UAAUR,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHU,WAAWV,IAAI,CAACe;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,kHAAkH;AAClH,yIAAyI;AACzI,OAAO,eAAeU,aAAajC,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;IACjH,MAAMkC,cAAc9C,oBAAoBC;IACxC,IAAI6C,gBAAgB,MAAM;QACxB,MAAM,IAAIzD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEyD,YAAYhD,KAAK,CAAC,GAAG,EAAEgD,YAAYxC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEwC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE3B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWhB;IACzC,IAAIiB,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMK,eAAejB,MAAMC;IAEjD,MAAM,EAAE6B,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBZ,OAAOC;IAC/D,MAAMgC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAMzC,KAAK0C,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,2FAA2F;AAC3F,gGAAgG;AAChG,OAAO,SAASK,mBAAmB5C,IAAgB;IACjD,OAAO;QACL6C,OAAO,CAACvD,OAAO4C,OAASD,aAAajC,MAAMV,OAAO4C,MAAMnC,SAASC;QACjE8C,WAAW,IAAMzC,iBAAiBL;IACpC;AACF"}

package/dist/esm/store/duckdb/open.d.ts

@@ -1,12 +1,13 @@
+import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "2";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
+export declare const SCHEMA_VERSION = "3";
+export type { OpenResult };
+interface DuckdbHandle {
+    instance: DuckDBInstance;
+    duckdb: DuckDBConnection;
 }
+export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
 export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;