sensemaking 0.19.1 → 0.20.0

package/dist/cjs/scan/reparse.js

@@ -16,11 +16,9 @@ _export(exports, {
         return reparseFiles;
     }
 });
-var _nodemodule = require("node:module");
 var _nodeos = require("node:os");
-var _nodepath = require("node:path");
 var _indexts = require("./index.js");
-var _workererrorts = require("./worker-error.js");
+var _poolts = require("./pool.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -173,34 +171,10 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
-// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.
-var _require = typeof require === 'undefined' ? (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString()) : require;
 // Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files
 // (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.
 var DEFAULT_THRESHOLD = 200;
 var DEFAULT_MAX_WORKERS = 8;
-// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
-// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
-var workerFile;
-function resolveWorkerFile() {
-    if (workerFile) return workerFile;
-    var load = (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString());
-    for(var _i = 0, _iter = [
-        '..',
-        '../..',
-        '../../..'
-    ]; _i < _iter.length; _i++){
-        var rel = _iter[_i];
-        try {
-            if (load("".concat(rel, "/package.json")).name === 'sensemaking') {
-                workerFile = (0, _nodepath.join)((0, _nodepath.dirname)(load.resolve("".concat(rel, "/package.json"))), 'dist', 'cjs', 'workers', 'parse.js');
-                return workerFile;
-            }
-        } catch (unused) {}
-    }
-    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
-}
 function featuresForFile(features, cfg, file) {
     return features.filter(function(feature) {
         return !feature.enabledForFile || feature.enabledForFile(cfg, file);
@@ -233,30 +207,19 @@ function reparseSerial(files, features, cfg, onParsed) {
     }
     return results;
 }
-// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
-// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.
-function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
+// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with
+// none given, opens an ephemeral ParsePool for this call alone and always closes it.
+function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
     return _async_to_generator(function() {
-        var Tinypool, workerData, pool, done, results, err;
+        var ephemeral, results, err;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    Tinypool = _require('tinypool').Tinypool;
-                    // cfg and the feature selection are constant for the dispatch, so they cross once per worker
-                    // as workerData. Features carry closures and cannot cross at all; only their names do.
-                    workerData = {
-                        cfg: cfg,
-                        featureNames: features.map(function(feature) {
-                            return feature.name;
-                        })
-                    };
-                    pool = new Tinypool({
-                        filename: resolveWorkerFile(),
-                        minThreads: maxWorkers,
-                        maxThreads: maxWorkers,
-                        workerData: workerData
-                    });
-                    done = 0;
+                    if (pool) return [
+                        2,
+                        pool.run(files, features, cfg, onParsed, maxWorkers)
+                    ];
+                    ephemeral = new _poolts.ParsePool();
                     _state.label = 1;
                 case 1:
                     _state.trys.push([
@@ -267,37 +230,13 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
                     ]);
                     return [
                         4,
-                        Promise.all(files.map(function(file) {
-                            return _async_to_generator(function() {
-                                var result;
-                                return _ts_generator(this, function(_state) {
-                                    switch(_state.label){
-                                        case 0:
-                                            return [
-                                                4,
-                                                pool.run(file)
-                                            ];
-                                        case 1:
-                                            result = _state.sent();
-                                            if (!result.ok) throw (0, _workererrorts.reviveError)(result.error);
-                                            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
-                                            return [
-                                                2,
-                                                {
-                                                    doc: result.doc,
-                                                    warnings: result.warnings
-                                                }
-                                            ];
-                                    }
-                                });
-                            })();
-                        }))
+                        ephemeral.run(files, features, cfg, onParsed, maxWorkers)
                     ];
                 case 2:
                     results = _state.sent();
                     return [
                         4,
-                        pool.destroy()
+                        ephemeral.close()
                     ];
                 case 3:
                     _state.sent();
@@ -307,11 +246,11 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
                     ];
                 case 4:
                     err = _state.sent();
-                    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse
+                    // Tear the pool down before rethrowing, but a close failure must not mask the parse
                     // failure that caused it -- not a `finally`, for that reason.
                     return [
                         4,
-                        pool.destroy().catch(function() {})
+                        ephemeral.close().catch(function() {})
                     ];
                 case 5:
                     _state.sent();
@@ -340,7 +279,7 @@ function reparseFiles(_0, _1, _2, _3, _4) {
                     ];
                     return [
                         4,
-                        reparsePooled(files, features, cfg, onParsed, maxWorkers)
+                        reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool)
                     ];
                 case 1:
                     _tmp = _state.sent();

package/dist/cjs/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\n// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n  // as workerData. Features carry closures and cannot cross at all; only their names do.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","_require","require","createRequire","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","Tinypool","workerData","pool","err","featureNames","map","filename","minThreads","maxThreads","Promise","all","result","run","ok","reviveError","error","doc","warnings","destroy","catch","knownColumns","options","threshold","docs","newColumns","seen","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QA6DgBA;eAAAA;;QA+CMC;eAAAA;;;0BA5GQ;sBACO;wBACP;uBAQJ;6BAEE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,qIAAqI;AACrI,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAmBnF,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAME,oBAAoB;AAC1B,IAAMC,sBAAsB;AAE5B,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOL,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCM,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAMO,SAASf,gBAAgBgB,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAe3B,gBAAgBgB,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB;;YAC1IC,UAGFC,YACAC,MACFR,MAIID,SAUCU;;;;oBAnBDH,WAAa9B,SAAS,YAAtB8B;oBACR,6FAA6F;oBAC7F,uFAAuF;oBACjFC,aAA8B;wBAAEhB,KAAAA;wBAAKmB,cAAcpB,SAASqB,GAAG,CAAC,SAACjB;mCAAYA,QAAQT,IAAI;;oBAAE;oBAC3FuB,OAAO,IAAIF,SAAS;wBAAEM,UAAU9B;wBAAqB+B,YAAYR;wBAAYS,YAAYT;wBAAYE,YAAAA;oBAAW;oBAClHP,OAAO;;;;;;;;;oBAIO;;wBAAMe,QAAQC,GAAG,CAC/BnB,MAAMc,GAAG,CAAC,SAAOnB;;oCACTyB;;;;4CAAU;;gDAAMT,KAAKU,GAAG,CAAC1B;;;4CAAzByB,SAAU;4CAChB,IAAI,CAACA,OAAOE,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACH,OAAOI,KAAK;4CAC9CvB,qBAAAA,+BAAAA,SAAW,EAAEE;4CACb;;gDAAO;oDAAEsB,KAAKL,OAAOK,GAAG;oDAAEC,UAAUN,OAAOM,QAAQ;gDAAC;;;;4BACtD;;;;oBANIxB,UAAU;oBAQhB;;wBAAMS,KAAKgB,OAAO;;;oBAAlB;oBACA;;wBAAOzB;;;oBACAU;oBACP,sFAAsF;oBACtF,8DAA8D;oBAC9D;;wBAAMD,KAAKgB,OAAO,GAAGC,KAAK,CAAC,YAAO;;;oBAAlC;oBACA,MAAMhB;;;;;;;IAEV;;AAIO,SAAelC;wCAAasB,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEmC,YAAiC,EAAE5B,QAAiC;YAAE6B,SAC1IA,oBACCA,qBADbC,WACAvB,YACAN,eAEA8B,MACAN,UACAO,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQT,KAAeU,cAC1BT,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMU;;;;;oBAX+IN,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqBhD;oBACjC0B,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBO,KAAKC,GAAG,CAACvD,qBAAqBwD,IAAAA,4BAAoB;yBAC3EvC,CAAAA,MAAMwC,MAAM,IAAIT,SAAQ,GAAxB/B;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO;;;2BAApD;;;;;;2BAAkET,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA5IC;oBAEA8B;oBACAN;oBACAO;oBACAC,OAAO,IAAIO,IAAIZ;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyC3B,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQuB,kBAAAA,KAAeU,2BAAVT;;4BAChBA,CAAAA,YAAAA,UAASrB,IAAI,OAAbqB,WAAc,qBAAGS;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAAClB,IAAImB,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACF,KAAKW,GAAG,CAACT,MAAM;wCAClBF,KAAKY,GAAG,CAACV;wCACTH,WAAW5B,IAAI,CAAC+B;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLJ,KAAK3B,IAAI,CAACoB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEO,MAAAA;4BAAMN,UAAAA;4BAAUO,YAAAA;wBAAW;;;;IACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","doc","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAoCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAtEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAC1BJ,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMK;;;;;oBAX+IR,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBS,KAAKC,GAAG,CAACnC,qBAAqBoC,IAAAA,4BAAoB;yBAC3E5B,CAAAA,MAAM6B,MAAM,IAAIX,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIQ,IAAId;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyCd,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQqB,kBAAAA,KAAeC,2BAAVJ;;4BAChBA,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGI;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACT,IAAIU,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACH,KAAKY,GAAG,CAACT,MAAM;wCAClBH,KAAKa,GAAG,CAACV;wCACTJ,WAAWhB,IAAI,CAACoB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLN,KAAKd,IAAI,CAACkB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEJ,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;wBAAW;;;;IACtC"}

package/dist/cjs/store/builder.d.cts

@@ -0,0 +1,11 @@
+import type { Config } from '../config/index.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export interface Builder {
+    build(): Promise<{
+        parsed: number;
+        warnings: string[];
+    }>;
+    close(): Promise<void>;
+    readonly poolsCreated: number;
+}
+export declare function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder;

package/dist/cjs/store/builder.d.ts

@@ -0,0 +1,11 @@
+import type { Config } from '../config/index.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export interface Builder {
+    build(): Promise<{
+        parsed: number;
+        warnings: string[];
+    }>;
+    close(): Promise<void>;
+    readonly poolsCreated: number;
+}
+export declare function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder;

package/dist/cjs/store/builder.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "createBuilder", {
+    enumerable: true,
+    get: function() {
+        return createBuilder;
+    }
+});
+var _poolts = require("../scan/pool.js");
+var _reconcilets = require("./reconcile.js");
+function createBuilder(conn, cfg, baseDir, dialect) {
+    var pool = new _poolts.ParsePool();
+    return {
+        build: function build() {
+            return (0, _reconcilets.reconcile)(conn, cfg, baseDir, dialect, pool);
+        },
+        close: function close() {
+            return pool.close();
+        },
+        get poolsCreated () {
+            return pool.poolsCreated;
+        }
+    };
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/builder.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reconcile } from './reconcile.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  build(): Promise<{ parsed: number; warnings: string[] }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// The pool is created at most once, lazily, on whichever build() call first needs it, and reused\n// by every later build() on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: () => reconcile(conn, cfg, baseDir, dialect, pool),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","conn","cfg","baseDir","dialect","pool","ParsePool","build","reconcile","close","poolsCreated"],"mappings":";;;;+BAiBgBA;;;eAAAA;;;sBAhBU;2BACA;AAenB,SAASA,cAAcC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB;IACrG,IAAMC,OAAO,IAAIC,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA;mBAAaC,IAAAA,sBAAS,EAACP,MAAMC,KAAKC,SAASC,SAASC;;QACpDI,OAAO,SAAPA;mBAAaJ,KAAKI,KAAK;;QACvB,IAAIC;YACF,OAAOL,KAAKK,YAAY;QAC1B;IACF;AACF"}

package/dist/cjs/store/duckdb/fieldStats.js

@@ -14,6 +14,9 @@ function _array_like_to_array(arr, len) {
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
     return arr2;
 }
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
 function _array_without_holes(arr) {
     if (Array.isArray(arr)) return _array_like_to_array(arr);
 }
@@ -48,9 +51,39 @@ function _iterable_to_array(iter) {
         return Array.from(iter);
     }
 }
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
 function _non_iterable_spread() {
     throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
 function _to_consumable_array(arr) {
     return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
 }
@@ -169,13 +202,30 @@ function classifyVariantType(name, field) {
     if (name === 'VARCHAR') return 'text';
     throw new Error('duckdb: variant_typeof() returned unrecognized type "'.concat(name, '" for column "').concat(field, '"'));
 }
+// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in
+// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same
+// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:
+// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking
+// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no
+// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300
+// columns runs in 7ms.
+var AGGREGATES_PER_QUERY = 16;
 function fieldStats(conn, columns, scopeWhere) {
     return _async_to_generator(function() {
-        var exprs, stmt, row;
+        var stats, start, chunk, exprs, stmt, row, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    exprs = columns.map(function(name, i) {
+                    stats = [];
+                    start = 0;
+                    _state.label = 1;
+                case 1:
+                    if (!(start < columns.length)) return [
+                        3,
+                        5
+                    ];
+                    chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);
+                    exprs = chunk.map(function(name, i) {
                         var quoted = (0, _sharedts.quoteIdent)(name);
                         return "COUNT(".concat(quoted, ") AS n").concat(i, ", string_agg(DISTINCT variant_typeof(").concat(quoted, "), '|') FILTER (WHERE ").concat(quoted, " IS NOT NULL) AS t").concat(i);
                     });
@@ -183,28 +233,55 @@ function fieldStats(conn, columns, scopeWhere) {
                         4,
                         conn.prepare("SELECT ".concat(exprs.join(', '), " FROM frontmatter ").concat(scopeWhere))
                     ];
-                case 1:
+                case 2:
                     stmt = _state.sent();
                     return [
                         4,
                         stmt.get()
                     ];
-                case 2:
+                case 3:
                     row = _state.sent();
-                    return [
-                        2,
-                        columns.map(function(name, i) {
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        _loop = function() {
+                            var _step_value = _sliced_to_array(_step.value, 2), i = _step_value[0], name = _step_value[1];
                             var _row_, _row_1;
                             var raw = ((_row_ = row["t".concat(i)]) !== null && _row_ !== void 0 ? _row_ : '').split('|').filter(Boolean);
                             var types = _to_consumable_array(new Set(raw.map(function(t) {
                                 return classifyVariantType(t, name);
                             }))).sort();
-                            return {
+                            stats.push({
                                 field: name,
                                 coverage: Number((_row_1 = row["n".concat(i)]) !== null && _row_1 !== void 0 ? _row_1 : 0),
                                 type: types.join(',')
-                            };
-                        })
+                            });
+                        };
+                        for(_iterator = chunk.entries()[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true)_loop();
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
+                    _state.label = 4;
+                case 4:
+                    start += AGGREGATES_PER_QUERY;
+                    return [
+                        3,
+                        1
+                    ];
+                case 5:
+                    return [
+                        2,
+                        stats
                     ];
             }
         });

package/dist/cjs/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => {\n    const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n    const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n    return { field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') };\n  });\n}\n"],"names":["fieldStats","classifyVariantType","name","field","test","startsWith","Error","conn","columns","scopeWhere","exprs","stmt","row","map","i","quoted","quoteIdent","prepare","join","get","raw","split","filter","Boolean","types","Set","t","sort","coverage","Number","type"],"mappings":";;;;+BAcsBA;;;eAAAA;;;wBAdK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG3B,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,AAAC,wDAA4EH,OAArBD,MAAK,kBAAsB,OAANC,OAAM;AACrG;AAIO,SAAeH,WAAWO,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;;YAChFC,OAIAC,MACAC;;;;oBALAF,QAAQF,QAAQK,GAAG,CAAC,SAACX,MAAMY;wBAC/B,IAAMC,SAASC,IAAAA,oBAAU,EAACd;wBAC1B,OAAO,AAAC,SAAuBY,OAAfC,QAAO,UAAiDA,OAAzCD,GAAE,yCAAsEC,OAA/BA,QAAO,0BAAmDD,OAA3BC,QAAO,sBAAsB,OAAFD;oBACpI;oBACa;;wBAAMP,KAAKU,OAAO,CAAC,AAAC,UAA8CR,OAArCC,MAAMQ,IAAI,CAAC,OAAM,sBAA+B,OAAXT;;;oBAAzEE,OAAO;oBACA;;wBAAMA,KAAKQ,GAAG;;;oBAArBP,MAAO;oBACb;;wBAAOJ,QAAQK,GAAG,CAAC,SAACX,MAAMY;gCACVF,OAEyBA;4BAFvC,IAAMQ,MAAM,EAAER,QAAAA,GAAG,CAAC,AAAC,IAAK,OAAFE,GAAI,cAAZF,mBAAAA,QAA2B,IAAIS,KAAK,CAAC,KAAKC,MAAM,CAACC;4BAC/D,IAAMC,QAAQ,AAAC,qBAAG,IAAIC,IAAIL,IAAIP,GAAG,CAAC,SAACa;uCAAMzB,oBAAoByB,GAAGxB;iCAASyB,IAAI;4BAC7E,OAAO;gCAAExB,OAAOD;gCAAM0B,UAAUC,QAAOjB,SAAAA,GAAG,CAAC,AAAC,IAAK,OAAFE,GAAI,cAAZF,oBAAAA,SAAgB;gCAAIkB,MAAMN,MAAMN,IAAI,CAAC;4BAAK;wBACnF;;;;IACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in\n// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same\n// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:\n// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking\n// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no\n// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300\n// columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["fieldStats","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","conn","columns","scopeWhere","stats","start","chunk","exprs","stmt","row","length","slice","map","i","quoted","quoteIdent","prepare","join","get","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type","entries"],"mappings":";;;;+BAuBsBA;;;eAAAA;;;wBAvBK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG3B,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,AAAC,wDAA4EH,OAArBD,MAAK,kBAAsB,OAANC,OAAM;AACrG;AAEA,qGAAqG;AACrG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,uBAAuB;AACvB,IAAMI,uBAAuB;AAItB,SAAeP,WAAWQ,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;;YAChFC,OACGC,OACDC,OACAC,OAIAC,MACAC,KACD,2BAAA,mBAAA,uBAAA,WAAA;;;;oBATDL;oBACGC,QAAQ;;;yBAAGA,CAAAA,QAAQH,QAAQQ,MAAM,AAAD;;;;oBACjCJ,QAAQJ,QAAQS,KAAK,CAACN,OAAOA,QAAQL;oBACrCO,QAAQD,MAAMM,GAAG,CAAC,SAACjB,MAAMkB;wBAC7B,IAAMC,SAASC,IAAAA,oBAAU,EAACpB;wBAC1B,OAAO,AAAC,SAAuBkB,OAAfC,QAAO,UAAiDA,OAAzCD,GAAE,yCAAsEC,OAA/BA,QAAO,0BAAmDD,OAA3BC,QAAO,sBAAsB,OAAFD;oBACpI;oBACa;;wBAAMZ,KAAKe,OAAO,CAAC,AAAC,UAA8Cb,OAArCI,MAAMU,IAAI,CAAC,OAAM,sBAA+B,OAAXd;;;oBAAzEK,OAAO;oBACA;;wBAAMA,KAAKU,GAAG;;;oBAArBT,MAAO;oBACR,kCAAA,2BAAA;;;4BAAA,mCAAA,iBAAOI,oBAAGlB;gCACCc,OAE6BA;4BAF3C,IAAMU,MAAM,EAAEV,QAAAA,GAAG,CAAC,AAAC,IAAK,OAAFI,GAAI,cAAZJ,mBAAAA,QAA2B,IAAIW,KAAK,CAAC,KAAKC,MAAM,CAACC;4BAC/D,IAAMC,QAAQ,AAAC,qBAAG,IAAIC,IAAIL,IAAIP,GAAG,CAAC,SAACa;uCAAM/B,oBAAoB+B,GAAG9B;iCAAS+B,IAAI;4BAC7EtB,MAAMuB,IAAI,CAAC;gCAAE/B,OAAOD;gCAAMiC,UAAUC,QAAOpB,SAAAA,GAAG,CAAC,AAAC,IAAK,OAAFI,GAAI,cAAZJ,oBAAAA,SAAgB;gCAAIqB,MAAMP,MAAMN,IAAI,CAAC;4BAAK;wBACvF;wBAJA,IAAK,YAAmBX,MAAMyB,OAAO,yBAAhC,6BAAA,QAAA,yBAAA;;wBAAA;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBARqC1B,SAASL;;;;;;oBAcrD;;wBAAOI;;;;IACT"}

package/dist/cjs/store/duckdb/lexical.d.cts

@@ -2,6 +2,7 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
+export declare function markContentStale(conn: Connection): void;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;

package/dist/cjs/store/duckdb/lexical.d.ts

@@ -2,6 +2,7 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
+export declare function markContentStale(conn: Connection): void;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;

package/dist/cjs/store/duckdb/lexical.js

@@ -14,6 +14,9 @@ _export(exports, {
     get createLexicalIndex () {
         return createLexicalIndex;
     },
+    get markContentStale () {
+        return markContentStale;
+    },
     get queryLexical () {
         return queryLexical;
     }
@@ -237,6 +240,22 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
+// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
+var ftsState = new WeakMap();
+function stateFor(conn) {
+    var state = ftsState.get(conn);
+    if (!state) {
+        state = {
+            stale: true
+        };
+        ftsState.set(conn, state);
+    }
+    return state;
+}
+function markContentStale(conn) {
+    stateFor(conn).stale = true;
+}
 // A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
 // (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
 function splitTerms(terms) {
@@ -480,15 +499,12 @@ function queryLexical(conn, terms, opts, state) {
     })();
 }
 function createLexicalIndex(conn) {
-    var state = {
-        stale: true
-    };
     return {
         query: function query(terms, opts) {
-            return queryLexical(conn, terms, opts, state);
+            return queryLexical(conn, terms, opts, stateFor(conn));
         },
         markStale: function markStale() {
-            state.stale = true;
+            return markContentStale(conn);
         }
     };
 }

package/dist/cjs/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),\n// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["createLexicalIndex","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","conn","state","stale","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QA8H9CA;eAAAA;;QAzBMC;eAAAA;;;wBApGK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACQ,IAAIC;QACtD,IAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAahB,eAAee,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAeC,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;;;oBAClB;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACJ,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKG,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAF,MAAMC,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASG,kBAAkBjB,KAAe,EAAEC,UAAoB;IAC9D,IAAMiB,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIrB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMiB,YAAYtB,MAAMuB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAexC,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAMyC,QAAN;gBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAgGkB,OAA9FxC,YAAY,CAACwC,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYd,IAAI,CAACgB;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUb,IAAI,CAAC;QACfe,WAAWf,IAAI,CAACgB;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAarB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMwB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAe5C,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAMyC,SAAN;oBACHN,WAAWZ,IAAI,CAAC,AAAC,GAAyDkB,OAAvDxC,YAAY,CAACwC,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYd,IAAI,CAACoB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUb,IAAI,CAAC;YACfe,WAAWf,IAAI,CAACoB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAevC,aAAa8B,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;;YAoBnGmB,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXtC,aAAtBC,OAAOC,YAKuBgB,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc1C,oBAAoBC;oBACxC,IAAIyC,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAY5C,KAAK,EAAC,OAAqDG,OAAhDyC,YAAYpC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQoC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXtC,cAAAA,WAAWP,QAAjCQ,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAeC,MAAMC;;;oBAA3B;;;oBAEgBI,qBAAAA,kBAAkBjB,OAAOC,aAAvD2B,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMxB,KAAK4B,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAASxD,mBAAmB+B,IAAgB;IACjD,IAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,SAAPA,MAAQlD,OAAOuC;mBAASjD,aAAa8B,MAAMpB,OAAOuC,MAAMlB;;QACxD8B,WAAW,SAAXA;YACE9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no\n// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently\n// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it\n// stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.\nconst ftsState = new WeakMap<Connection, FtsIndexState>();\nfunction stateFor(conn: Connection): FtsIndexState {\n  let state = ftsState.get(conn);\n  if (!state) {\n    state = { stale: true };\n    ftsState.set(conn, state);\n  }\n  return state;\n}\n\n// Called whenever this store's reconcileContent changes `content`, so the next lexical query\n// rebuilds instead of searching stale content.\nexport function markContentStale(conn: Connection): void {\n  stateFor(conn).stale = true;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token\n// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!hasUnspacedRun(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only\n// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's\n    // default 571-word list would otherwise drop common words (e.g. \"and\") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.\n// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).\n// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches\n// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, stateFor(conn)),\n    markStale: () => markContentStale(conn),\n  };\n}\n"],"names":["createLexicalIndex","markContentStale","queryLexical","FIELDS","FIELD_WEIGHT","title","summary","text","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","ftsState","WeakMap","stateFor","conn","state","get","stale","set","splitTerms","words","substrings","_m","inner","phrase","length","push","hasUnspacedRun","split","w","tok","ensureFtsFresh","exec","withTransaction","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","opts","stmt","unsupported","whereJoin","whereCond","scopeCond","limit","sql","SenseError","prepare","all","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;;;;;;;;;;;;QAgJ9CA;eAAAA;;QA/FAC;eAAAA;;QAsEMC;eAAAA;;;wBAtHK;yBACI;6BACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,uGAAuG;AACvG,iGAAiG;AACjG,IAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,0GAA0G;AAC1G,0IAA0I;AAC1I,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAMA,6FAA6F;AAC7F,0GAA0G;AAC1G,IAAMC,WAAW,IAAIC;AACrB,SAASC,SAASC,IAAgB;IAChC,IAAIC,QAAQJ,SAASK,GAAG,CAACF;IACzB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;QAAK;QACtBN,SAASO,GAAG,CAACJ,MAAMC;IACrB;IACA,OAAOA;AACT;AAIO,SAAStB,iBAAiBqB,IAAgB;IAC/CD,SAASC,MAAMG,KAAK,GAAG;AACzB;AAEA,uHAAuH;AACvH,6IAA6I;AAC7I,SAASE,WAAWf,KAAa;IAC/B,IAAMgB,QAAkB,EAAE;IAC1B,IAAMC,aAAuB,EAAE;IAC/B,IAAMhB,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACgB,IAAIC;QACtD,IAAMC,SAASD,MAAMb,IAAI;QACzB,IAAIc,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAACG,IAAAA,yBAAc,EAACH,SAAS;gBAAK,kCAAA,2BAAA;;gBAAL,QAAK,YAAWA,OAAOI,KAAK,CAAC,2BAAxB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,IAAN;oBAAgC,IAAIA,EAAEJ,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACG;;;gBAA7D;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAA+D;QACjG,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaxB,eAAeuB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAME,MAAN;YACH,IAAIA,IAAIL,MAAM,KAAK,GAAG;YACtB,IAAIE,IAAAA,yBAAc,EAACG,MAAMT,WAAWK,IAAI,CAACI;iBACpCV,MAAMM,IAAI,CAACI;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,YAAAA;IAAW;AAC7B;AAEA,6HAA6H;AAC7H,wIAAwI;AACxI,SAAeU,eAAejB,IAAgB,EAAEC,KAAoB;;;;;oBAClE,IAAI,CAACA,MAAME,KAAK,EAAE;;;oBAClB;;wBAAMH,KAAKkB,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMC,IAAAA,8BAAe,EAACnB,MAAM;;;;;4CAC1B,6HAA6H;4CAC7H,0JAA0J;4CAC1J;;gDAAMA,KAAKkB,IAAI,CAAC;;;4CAAhB;;;;;;4BACF;;;;oBAJA;oBAKAjB,MAAME,KAAK,GAAG;;;;;;IAChB;;AAEA,qKAAqK;AACrK,+OAA+O;AAC/O,SAASiB,kBAAkBd,KAAe,EAAEC,UAAoB;IAC9D,IAAMc,aAAuB,EAAE;IAC/B,IAAMC,YAAsB,EAAE;IAC9B,IAAMC,cAAyB,EAAE;IACjC,IAAMC,aAAwB,EAAE;IAEhC,IAAIlB,MAAMK,MAAM,GAAG,GAAG;QACpB,IAAMc,YAAYnB,MAAMoB,IAAI,CAAC;YACxB,kCAAA,2BAAA;;YAAL,QAAK,YAAe7C,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;gBAAvB,IAAM8C,QAAN;gBACHN,WAAWT,IAAI,CAAC,AAAC,GAAgGe,OAA9F7C,YAAY,CAAC6C,MAAM,EAAC,6EAAgF,OAANA,OAAM;gBACvHJ,YAAYX,IAAI,CAACa;YACnB;;YAHK;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;QAILH,UAAUV,IAAI,CAAC;QACfY,WAAWZ,IAAI,CAACa;IAClB;QAEK,mCAAA,4BAAA;;QAAL,QAAK,aAAalB,+BAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAyB;YAAzB,IAAMqB,MAAN;YACH,IAAMC,SAASD,IAAIE,WAAW;gBACzB,mCAAA,4BAAA;;gBAAL,QAAK,aAAejD,2BAAf,UAAA,8BAAA,SAAA,0BAAA,kCAAuB;oBAAvB,IAAM8C,SAAN;oBACHN,WAAWT,IAAI,CAAC,AAAC,GAAyDe,OAAvD7C,YAAY,CAAC6C,OAAM,EAAC,qCAAyC,OAANA,QAAM;oBAChFJ,YAAYX,IAAI,CAACiB;gBACnB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAILP,UAAUV,IAAI,CAAC;YACfY,WAAWZ,IAAI,CAACiB;QAClB;;QARK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ,AAAC,qBAAGV,oBAAa,qBAAGC;IAAY;AACvH;AAIO,SAAe5C,aAAaoB,IAAgB,EAAEV,KAAa,EAAE4C,IAAyB,EAAEjC,KAAoB;;YAoBnGkC,OAnBRC,aAIEC,WAAWC,WAAWC,WAAWC,OACXnC,aAAtBC,OAAOC,YAKuBa,oBAA9BW,UAAUC,SAASC,QACrBQ,KAOAN;;;;oBAlBAC,cAAc/C,oBAAoBC;oBACxC,IAAI8C,gBAAgB,MAAM;wBACxB,MAAM,IAAIM,oBAAU,CAAC,4BAA4B,AAAC,6CAAkEN,OAAvBA,YAAYjD,KAAK,EAAC,OAAqDG,OAAhD8C,YAAYzC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBACnL;oBACQyC,YAA2CH,KAA3CG,WAAWC,YAAgCJ,KAAhCI,WAAWC,YAAqBL,KAArBK,WAAWC,QAAUN,KAAVM;oBACXnC,cAAAA,WAAWf,QAAjCgB,QAAsBD,YAAtBC,OAAOC,aAAeF,YAAfE;oBACf,IAAID,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG;;;;yBAE/CL,CAAAA,MAAMK,MAAM,GAAG,CAAA,GAAfL;;;;oBAAkB;;wBAAMW,eAAejB,MAAMC;;;oBAA3B;;;oBAEgBmB,qBAAAA,kBAAkBd,OAAOC,aAAvDwB,WAA8BX,mBAA9BW,UAAUC,UAAoBZ,mBAApBY,SAASC,SAAWb,mBAAXa;oBACrBQ,MAAM,AAAC,wEAGTJ,OAFgCN,UAAS,sCAGnCC,OADNK,WAAU,gBAEVC,OADMN,SAAQ,UACDO,OAAbD,WAAU,KAAa,OAAVC,WAAU;oBAEd;;wBAAMvC,KAAK2C,OAAO,CAACF;;;oBAA1BN,OAAO;oBACL;;wBAAMA,CAAAA,QAAAA,MAAKS,GAAG,OAART,OAAAA,AAAS,qBAAGF;4BAAQO;;;;oBAAlC;;wBAAQ;;;;IACV;;AAIO,SAAS9D,mBAAmBsB,IAAgB;IACjD,OAAO;QACL6C,OAAO,SAAPA,MAAQvD,OAAO4C;mBAAStD,aAAaoB,MAAMV,OAAO4C,MAAMnC,SAASC;;QACjE8C,WAAW,SAAXA;mBAAiBnE,iBAAiBqB;;IACpC;AACF"}

package/dist/cjs/store/duckdb/open.d.cts

@@ -1,12 +1,13 @@
+import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "2";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
+export declare const SCHEMA_VERSION = "3";
+export type { OpenResult };
+interface DuckdbHandle {
+    instance: DuckDBInstance;
+    duckdb: DuckDBConnection;
 }
+export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
 export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/duckdb/open.d.ts

@@ -1,12 +1,13 @@
+import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
 import type { ResolvedConfig } from '../../config/index.js';
-import type { Store } from '../types.js';
+import type { OpenResult } from '../open.js';
+import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.duckdb";
-export declare const SCHEMA_VERSION = "2";
-export interface OpenResult {
-    store: Store;
-    cfg: ResolvedConfig;
-    dbPath: string;
-    parsed: number;
-    warnings: string[];
+export declare const SCHEMA_VERSION = "3";
+export type { OpenResult };
+interface DuckdbHandle {
+    instance: DuckDBInstance;
+    duckdb: DuckDBConnection;
 }
+export declare const duckdbOpenDialect: OpenDialect<DuckdbHandle>;
 export declare function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult>;