sensemaking 0.27.1 → 0.28.0

package/dist/cjs/store/duckdb/native.js

@@ -9,28 +9,172 @@ function _export(target, all) {
     });
 }
 _export(exports, {
-    get DUCKDB_INSTALL_SPEC () {
-        return DUCKDB_INSTALL_SPEC;
-    },
     get DUCKDB_PACKAGE () {
         return DUCKDB_PACKAGE;
     },
     get duckdbApi () {
         return duckdbApi;
+    },
+    get duckdbNative () {
+        return duckdbNative;
     }
 });
 var _nativets = require("../native.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
 var DUCKDB_PACKAGE = '@duckdb/node-api';
-var DUCKDB_INSTALL_SPEC = "".concat(DUCKDB_PACKAGE, "@1.5.6-r.1");
-var DUCKDB = {
-    store: 'duckdb',
-    pkg: DUCKDB_PACKAGE,
-    sizeHint: '~110MB',
-    installSpec: DUCKDB_INSTALL_SPEC
-};
-var duckdbApiPromise;
+var duckdbNativePromise;
+function duckdbNative() {
+    if (!duckdbNativePromise) {
+        var descriptor = {
+            store: 'duckdb',
+            pkg: DUCKDB_PACKAGE,
+            sizeHint: '~110MB',
+            range: (0, _nativets.peerRange)(DUCKDB_PACKAGE)
+        };
+        duckdbNativePromise = (0, _nativets.loadOrInstall)(descriptor, (0, _nativets.packageNodeModules)());
+    }
+    return duckdbNativePromise;
+}
 function duckdbApi() {
-    if (!duckdbApiPromise) duckdbApiPromise = (0, _nativets.loadOrInstall)(DUCKDB, (0, _nativets.packageNodeModules)());
-    return duckdbApiPromise;
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        duckdbNative()
+                    ];
+                case 1:
+                    return [
+                        2,
+                        _state.sent().api
+                    ];
+            }
+        });
+    })();
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\nexport const DUCKDB_INSTALL_SPEC = `${DUCKDB_PACKAGE}@1.5.6-r.1`;\n\nconst DUCKDB: NativeDescriptor = {\n  store: 'duckdb',\n  pkg: DUCKDB_PACKAGE,\n  sizeHint: '~110MB',\n  installSpec: DUCKDB_INSTALL_SPEC,\n};\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["DUCKDB_INSTALL_SPEC","DUCKDB_PACKAGE","duckdbApi","DUCKDB","store","pkg","sizeHint","installSpec","duckdbApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAGaA;eAAAA;;QADAC;eAAAA;;QAcGC;eAAAA;;;wBAhByD;AAElE,IAAMD,iBAAiB;AACvB,IAAMD,sBAAsB,AAAC,GAAiB,OAAfC,gBAAe;AAErD,IAAME,SAA2B;IAC/BC,OAAO;IACPC,KAAKJ;IACLK,UAAU;IACVC,aAAaP;AACf;AAEA,IAAIQ;AAIG,SAASN;IACd,IAAI,CAACM,kBAAkBA,mBAAmBC,IAAAA,uBAAa,EAAoCN,QAAQO,IAAAA,4BAAkB;IACrH,OAAOF;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, type NativeModule, packageNodeModules, peerRange } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\ntype DuckdbApi = typeof import('@duckdb/node-api');\n\nlet duckdbNativePromise: Promise<NativeModule<DuckdbApi>> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api must go through: after a fresh install in this process, Node keeps returning\n// the pre-install \"not found\" miss to any new bare-specifier lookup of the same path, so caching the resolved module here means only the first caller ever resolves it.\nexport function duckdbNative(): Promise<NativeModule<DuckdbApi>> {\n  if (!duckdbNativePromise) {\n    const descriptor: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB', range: peerRange(DUCKDB_PACKAGE) };\n    duckdbNativePromise = loadOrInstall<DuckdbApi>(descriptor, packageNodeModules());\n  }\n  return duckdbNativePromise;\n}\n\nexport async function duckdbApi(): Promise<DuckdbApi> {\n  return (await duckdbNative()).api;\n}\n"],"names":["DUCKDB_PACKAGE","duckdbApi","duckdbNative","duckdbNativePromise","descriptor","store","pkg","sizeHint","range","peerRange","loadOrInstall","packageNodeModules","api"],"mappings":";;;;;;;;;;;QAEaA;eAAAA;;QAgBSC;eAAAA;;QARNC;eAAAA;;;wBAVuF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEhG,IAAMF,iBAAiB;AAI9B,IAAIG;AAIG,SAASD;IACd,IAAI,CAACC,qBAAqB;QACxB,IAAMC,aAA+B;YAAEC,OAAO;YAAUC,KAAKN;YAAgBO,UAAU;YAAUC,OAAOC,IAAAA,mBAAS,EAACT;QAAgB;QAClIG,sBAAsBO,IAAAA,uBAAa,EAAYN,YAAYO,IAAAA,4BAAkB;IAC/E;IACA,OAAOR;AACT;AAEO,SAAeF;;;;;oBACZ;;wBAAMC;;;oBAAd;;wBAAQ,cAAsBU,GAAG;;;;IACnC"}

package/dist/cjs/store/duckdb/open.js

@@ -25,11 +25,13 @@ _export(exports, {
 var _typests = require("../../embed/types.js");
 var _errorsts = require("../../errors.js");
 var _indexts = require("../../features/index.js");
+var _enginets = require("../engine.js");
+var _nativets = require("../native.js");
 var _opents = require("../open.js");
 var _sharedts = require("../shared.js");
 var _connectionts = require("./connection.js");
 var _lexicalts = require("./lexical.js");
-var _nativets = require("./native.js");
+var _nativets1 = require("./native.js");
 var _reconcilets = require("./reconcile.js");
 var _sqlfunctionsts = require("./sql-functions.js");
 var _storets = require("./store.js");
@@ -358,7 +360,7 @@ function connect(dbPath, _cfg, options) {
                     ]);
                     return [
                         4,
-                        (0, _nativets.duckdbApi)()
+                        (0, _nativets1.duckdbApi)()
                     ];
                 case 1:
                     DuckDBInstance = _state.sent().DuckDBInstance;
@@ -369,7 +371,7 @@ function connect(dbPath, _cfg, options) {
                 case 2:
                     err = _state.sent();
                     if (_instanceof(err, _errorsts.SenseError)) throw err;
-                    throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" needs the '.concat(_nativets.DUCKDB_PACKAGE, " package (").concat(err.message, ")"));
+                    throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" needs the '.concat(_nativets1.DUCKDB_PACKAGE, " package (").concat(err.message, ")"));
                 case 3:
                     _state.trys.push([
                         3,
@@ -448,6 +450,150 @@ function connect(dbPath, _cfg, options) {
         });
     })();
 }
+function duckdbEngineIdentity() {
+    return _async_to_generator(function() {
+        var _tmp;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _tmp = [
+                        _nativets1.DUCKDB_PACKAGE
+                    ];
+                    return [
+                        4,
+                        (0, _nativets1.duckdbNative)()
+                    ];
+                case 1:
+                    return [
+                        2,
+                        _enginets.engineIdentity.apply(void 0, _tmp.concat([
+                            _state.sent().version
+                        ]))
+                    ];
+            }
+        });
+    })();
+}
+// The real schema, index build and lexical query over store/engine.ts's fixture, in memory.
+function checkDuckdbEngine() {
+    return _async_to_generator(function() {
+        var _ref, api, version, instance, handle, _tmp, conn;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        (0, _nativets1.duckdbNative)()
+                    ];
+                case 1:
+                    _ref = _state.sent(), api = _ref.api, version = _ref.version;
+                    return [
+                        4,
+                        api.DuckDBInstance.create(':memory:')
+                    ];
+                case 2:
+                    instance = _state.sent();
+                    _tmp = {
+                        instance: instance
+                    };
+                    return [
+                        4,
+                        instance.connect()
+                    ];
+                case 3:
+                    handle = (_tmp.duckdb = _state.sent(), _tmp);
+                    _state.label = 4;
+                case 4:
+                    _state.trys.push([
+                        4,
+                        ,
+                        9,
+                        11
+                    ]);
+                    conn = (0, _connectionts.createConnection)(handle.duckdb);
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)')
+                    ];
+                case 5:
+                    _state.sent();
+                    return [
+                        4,
+                        ensureCoreSchema(handle, conn)
+                    ];
+                case 6:
+                    _state.sent();
+                    return [
+                        4,
+                        conn.runBatch('INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)', _enginets.ENGINE_CHECK_DOCS.map(function(doc) {
+                            return [
+                                doc.path,
+                                doc.title,
+                                doc.summary,
+                                doc.text
+                            ];
+                        }))
+                    ];
+                case 7:
+                    _state.sent();
+                    return [
+                        4,
+                        (0, _enginets.checkEngine)('duckdb', (0, _enginets.engineIdentity)(_nativets1.DUCKDB_PACKAGE, version), (0, _enginets.npmRemedy)(_nativets1.DUCKDB_PACKAGE, (0, _nativets.peerRange)(_nativets1.DUCKDB_PACKAGE)), function(terms) {
+                            return _async_to_generator(function() {
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                (0, _lexicalts.prepareFts)(conn)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (0, _lexicalts.queryLexical)(conn, terms, {
+                                                    whereJoin: '',
+                                                    whereCond: '',
+                                                    scopeCond: '',
+                                                    limit: 10
+                                                })
+                                            ];
+                                        case 2:
+                                            return [
+                                                2,
+                                                _state.sent().map(function(hit) {
+                                                    return hit.path;
+                                                })
+                                            ];
+                                    }
+                                });
+                            })();
+                        })
+                    ];
+                case 8:
+                    _state.sent();
+                    return [
+                        3,
+                        11
+                    ];
+                case 9:
+                    return [
+                        4,
+                        close(handle)
+                    ];
+                case 10:
+                    _state.sent();
+                    return [
+                        7
+                    ];
+                case 11:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 var duckdbOpenDialect = {
     filename: DB_FILENAME,
     schemaVersion: SCHEMA_VERSION,
@@ -460,6 +606,8 @@ var duckdbOpenDialect = {
     isLocked: function isLocked(err) {
         return /Could not set lock on file|being used by another process/.test(err.message);
     },
+    engineIdentity: duckdbEngineIdentity,
+    checkEngine: checkDuckdbEngine,
     ensureCoreSchema: ensureCoreSchema,
     ensureFeatureSchema: ensureFeatureSchema,
     prepareConnection: prepareConnection,

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '6';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureCoreSchema(_handle: DuckdbHandle, conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\nasync function ensureFeatureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n}\n\nasync function prepareConnection(handle: DuckdbHandle, _conn: Connection, _cfg: Config): Promise<void> {\n  await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  prepareConnection,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureCoreSchema","_handle","conn","exec","getMeta","setMeta","ensureFeatureSchema","cfg","feature","activeFeatures","name","STORE_DIMS","schema","prepareConnection","handle","_conn","_cfg","createVectorWriteStage","duckdb","close","disconnectSync","instance","closeSync","connect","dbPath","options","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","observational","access_mode","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","prepareLexical","prepareFts","assertLexicalReady","assertFtsReady","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAiBaA;eAAAA;;QAGAC;eAAAA;;QAgFAC;eAAAA;;QAkBSC;eAAAA;;;uBApHK;wBACA;uBACI;sBAEC;wBACC;4BAEA;yBACU;wBACD;2BACZ;8BACI;uBACN;yBACW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEhC,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,SAAeG,iBAAiBC,OAAqB,EAAEC,IAAgB;;;;;oBACrE;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACK;;wBAAMC,IAAAA,iBAAO,EAACF,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMG,IAAAA,iBAAO,EAACH,MAAM,kBAAkBL;;;oBAAtC;;;;;;;;IACxD;;AAEA,SAAeS,oBAAoBL,OAAqB,EAAEC,IAAgB,EAAEK,GAAW;;YAChF,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,uBAAc,EAACF;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQE,IAAI,KAAK,OAAM,GAAvBF;;;;oBACF;;wBAAMN,KAAKC,IAAI,CAAC,AAAC,qKAA+K,OAAXQ,mBAAU,EAAC;;;oBAAhM;oBACA;;;;;oBAEF;;wBAAMH,QAAQI,MAAM,CAACV;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IASP;;AAEA,SAAeW,kBAAkBC,MAAoB,EAAEC,KAAiB,EAAEC,IAAY;;;;;oBACpF;;wBAAMC,IAAAA,iCAAsB,EAACH,OAAOI,MAAM,EAAEP,mBAAU;;;oBAAtD;;;;;;IACF;;AAEA,iFAAiF;AACjF,SAAeQ,MAAML,MAAoB;;;YACvCA,OAAOI,MAAM,CAACE,cAAc;YAC5BN,OAAOO,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAER,IAAoB,EAAES,OAA0D;;YAGjHC,gBAGKC,KAKLT,QACAG,UAMKM,MAQDzB,MAECyB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQC,QAAQQ,aAAa,GAAG;4BAAEC,aAAa;wBAAY,IAAI;4BAAEC,+BAA+B;wBAAS;;;oBAFhJ,4HAA4H;oBAC5H,4KAA4K;oBAC5Kd,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BL,SAAS;;;;;;oBACFS;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMK,IAAAA,iCAAiB,EAAClB;;;oBAAxB;oBACMhB,OAAOmC,IAAAA,8BAAgB,EAACnB;oBAC9B;;wBAAO;4BAAEJ,QAAQ;gCAAEO,UAAAA;gCAAUH,QAAAA;4BAAO;4BAAGhB,MAAAA;wBAAK;;;oBACrCyB;oBACP;;wBAAMR,MAAM;4BAAEE,UAAAA;4BAAUH,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMS;;;;;;;IAEV;;AAGO,IAAM7B,oBAA+C;IAC1DwC,UAAU1C;IACV2C,eAAe1C;IACf2C,kBAAkBC,0BAAa;IAC/BlB,SAAAA;IACAJ,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWf;eAAQ,2DAA2DgB,IAAI,CAAChB,IAAII,OAAO;;IAC9F/B,kBAAAA;IACAM,qBAAAA;IACAO,mBAAAA;IACA+B,gBAAgBC,qBAAU;IAC1BC,oBAAoBC,yBAAc;IAClCC,aAAa,SAAbA,YAAclC,QAAQZ;eAAS8C,IAAAA,oBAAW,EAAClC,OAAOO,QAAQ,EAAEP,OAAOI,MAAM,EAAEhB;;AAC7E;AAEO,SAAeH,WAAWQ,GAAmB,EAAEkB,OAA6B;;;YACjF;;gBAAOwB,IAAAA,uBAAe,EAAC1C,KAAKT,mBAAmB2B;;;IACjD"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport { ENGINE_CHECK_DOCS, engineIdentity, npmRemedy, checkEngine as runEngineCheck } from '../engine.ts';\nimport { peerRange } from '../native.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { assertFtsReady, prepareFts, queryLexical } from './lexical.ts';\nimport { DUCKDB_PACKAGE, duckdbApi, duckdbNative } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\nimport { createVectorWriteStage } from './vectors.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '6';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): build prepares the persistent FTS index over it,\n// while contains() verification reads the table directly.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureCoreSchema(_handle: DuckdbHandle, conn: Connection): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\nasync function ensureFeatureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n}\n\nasync function prepareConnection(handle: DuckdbHandle, _conn: Connection, _cfg: Config): Promise<void> {\n  await createVectorWriteStage(handle.duckdb, STORE_DIMS);\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, options.observational ? { access_mode: 'READ_ONLY' } : { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\nasync function duckdbEngineIdentity(): Promise<string> {\n  return engineIdentity(DUCKDB_PACKAGE, (await duckdbNative()).version);\n}\n\n// The real schema, index build and lexical query over store/engine.ts's fixture, in memory.\nasync function checkDuckdbEngine(): Promise<void> {\n  const { api, version } = await duckdbNative();\n  const instance = await api.DuckDBInstance.create(':memory:');\n  const handle = { instance, duckdb: await instance.connect() };\n  try {\n    const conn = createConnection(handle.duckdb);\n    await conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)');\n    await ensureCoreSchema(handle, conn);\n    await conn.runBatch(\n      'INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)',\n      ENGINE_CHECK_DOCS.map((doc) => [doc.path, doc.title, doc.summary, doc.text])\n    );\n    await runEngineCheck('duckdb', engineIdentity(DUCKDB_PACKAGE, version), npmRemedy(DUCKDB_PACKAGE, peerRange(DUCKDB_PACKAGE)), async (terms) => {\n      await prepareFts(conn);\n      return (await queryLexical(conn, terms, { whereJoin: '', whereCond: '', scopeCond: '', limit: 10 })).map((hit) => hit.path);\n    });\n  } finally {\n    await close(handle);\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  engineIdentity: duckdbEngineIdentity,\n  checkEngine: checkDuckdbEngine,\n  ensureCoreSchema,\n  ensureFeatureSchema,\n  prepareConnection,\n  prepareLexical: prepareFts,\n  assertLexicalReady: assertFtsReady,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect, options);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureCoreSchema","_handle","conn","exec","getMeta","setMeta","ensureFeatureSchema","cfg","feature","activeFeatures","name","STORE_DIMS","schema","prepareConnection","handle","_conn","_cfg","createVectorWriteStage","duckdb","close","disconnectSync","instance","closeSync","connect","dbPath","options","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","observational","access_mode","storage_compatibility_version","registerFunctions","createConnection","duckdbEngineIdentity","duckdbNative","engineIdentity","version","checkDuckdbEngine","api","runBatch","ENGINE_CHECK_DOCS","map","doc","path","title","summary","text","runEngineCheck","npmRemedy","peerRange","terms","prepareFts","queryLexical","whereJoin","whereCond","scopeCond","limit","hit","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","checkEngine","prepareLexical","assertLexicalReady","assertFtsReady","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAmBaA;eAAAA;;QAGAC;eAAAA;;QA0GAC;eAAAA;;QAoBSC;eAAAA;;;uBAlJK;wBACA;uBACI;wBAC6D;wBAClE;sBAEM;wBACC;4BAEA;yBACwB;yBACD;2BAC1B;8BACI;uBACN;yBACW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEhC,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,iGAAiG;AACjG,0DAA0D;AAC1D,gGAAgG;AAChG,SAAeG,iBAAiBC,OAAqB,EAAEC,IAAgB;;;;;oBACrE;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACK;;wBAAMC,IAAAA,iBAAO,EAACF,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMG,IAAAA,iBAAO,EAACH,MAAM,kBAAkBL;;;oBAAtC;;;;;;;;IACxD;;AAEA,SAAeS,oBAAoBL,OAAqB,EAAEC,IAAgB,EAAEK,GAAW;;YAChF,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,uBAAc,EAACF;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQE,IAAI,KAAK,OAAM,GAAvBF;;;;oBACF;;wBAAMN,KAAKC,IAAI,CAAC,AAAC,qKAA+K,OAAXQ,mBAAU,EAAC;;;oBAAhM;oBACA;;;;;oBAEF;;wBAAMH,QAAQI,MAAM,CAACV;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IASP;;AAEA,SAAeW,kBAAkBC,MAAoB,EAAEC,KAAiB,EAAEC,IAAY;;;;;oBACpF;;wBAAMC,IAAAA,iCAAsB,EAACH,OAAOI,MAAM,EAAEP,mBAAU;;;oBAAtD;;;;;;IACF;;AAEA,iFAAiF;AACjF,SAAeQ,MAAML,MAAoB;;;YACvCA,OAAOI,MAAM,CAACE,cAAc;YAC5BN,OAAOO,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAER,IAAoB,EAAES,OAA0D;;YAGjHC,gBAGKC,KAKLT,QACAG,UAMKM,MAQDzB,MAECyB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,oBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,yBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQC,QAAQQ,aAAa,GAAG;4BAAEC,aAAa;wBAAY,IAAI;4BAAEC,+BAA+B;wBAAS;;;oBAFhJ,4HAA4H;oBAC5H,4KAA4K;oBAC5Kd,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BL,SAAS;;;;;;oBACFS;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMK,IAAAA,iCAAiB,EAAClB;;;oBAAxB;oBACMhB,OAAOmC,IAAAA,8BAAgB,EAACnB;oBAC9B;;wBAAO;4BAAEJ,QAAQ;gCAAEO,UAAAA;gCAAUH,QAAAA;4BAAO;4BAAGhB,MAAAA;wBAAK;;;oBACrCyB;oBACP;;wBAAMR,MAAM;4BAAEE,UAAAA;4BAAUH,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMS;;;;;;;IAEV;;AAEA,SAAeW;;;;;;;wBACSR,yBAAc;;oBAAG;;wBAAMS,IAAAA,uBAAY;;;oBAAzD;;wBAAOC,wBAAc;4BAAkB,cAAsBC,OAAO;;;;;IACtE;;AAEA,4FAA4F;AAC5F,SAAeC;;YACY,MAAjBC,KAAKF,SACPpB,UACAP,cAEEZ;;;;oBAJiB;;wBAAMqC,IAAAA,uBAAY;;;oBAAlB,OAAA,eAAjBI,MAAiB,KAAjBA,KAAKF,UAAY,KAAZA;oBACI;;wBAAME,IAAIjB,cAAc,CAACM,MAAM,CAAC;;;oBAA3CX,WAAW;;wBACAA,UAAAA;;oBAAkB;;wBAAMA,SAASE,OAAO;;;oBAAnDT,eAAqBI,SAAQ;;;;;;;;;oBAE3BhB,OAAOmC,IAAAA,8BAAgB,EAACvB,OAAOI,MAAM;oBAC3C;;wBAAMhB,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,iBAAiBc,QAAQZ;;;oBAA/B;oBACA;;wBAAMA,KAAK0C,QAAQ,CACjB,0EACAC,2BAAiB,CAACC,GAAG,CAAC,SAACC;mCAAQ;gCAACA,IAAIC,IAAI;gCAAED,IAAIE,KAAK;gCAAEF,IAAIG,OAAO;gCAAEH,IAAII,IAAI;6BAAC;;;;oBAF7E;oBAIA;;wBAAMC,IAAAA,qBAAc,EAAC,UAAUZ,IAAAA,wBAAc,EAACV,yBAAc,EAAEW,UAAUY,IAAAA,mBAAS,EAACvB,yBAAc,EAAEwB,IAAAA,mBAAS,EAACxB,yBAAc,IAAI,SAAOyB;;;;;4CACnI;;gDAAMC,IAAAA,qBAAU,EAACtD;;;4CAAjB;4CACQ;;gDAAMuD,IAAAA,uBAAY,EAACvD,MAAMqD,OAAO;oDAAEG,WAAW;oDAAIC,WAAW;oDAAIC,WAAW;oDAAIC,OAAO;gDAAG;;;4CAAjG;;gDAAQ,cAA6Ff,GAAG,CAAC,SAACgB;2DAAQA,IAAId,IAAI;;;;;4BAC5H;;;;oBAHA;;;;;;oBAKA;;wBAAM7B,MAAML;;;oBAAZ;;;;;;;;;;IAEJ;;AAGO,IAAMhB,oBAA+C;IAC1DiE,UAAUnE;IACVoE,eAAenE;IACfoE,kBAAkBC,0BAAa;IAC/B3C,SAAAA;IACAJ,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFgD,UAAU,SAAVA,SAAWxC;eAAQ,2DAA2DyC,IAAI,CAACzC,IAAII,OAAO;;IAC9FS,gBAAgBF;IAChB+B,aAAa3B;IACb1C,kBAAAA;IACAM,qBAAAA;IACAO,mBAAAA;IACAyD,gBAAgBd,qBAAU;IAC1Be,oBAAoBC,yBAAc;IAClCC,aAAa,SAAbA,YAAc3D,QAAQZ;eAASuE,IAAAA,oBAAW,EAAC3D,OAAOO,QAAQ,EAAEP,OAAOI,MAAM,EAAEhB;;AAC7E;AAEO,SAAeH,WAAWQ,GAAmB,EAAEkB,OAA6B;;;YACjF;;gBAAOiD,IAAAA,uBAAe,EAACnE,KAAKT,mBAAmB2B;;;IACjD"}

package/dist/cjs/store/duckdb/ordered-bm25.d.cts

@@ -1,4 +1,4 @@
 import type { Connection } from '../types.js';
-export declare const FTS_STEMMER = "porter";
-export declare function nativeQueryTerms(conn: Connection, query: string): Promise<string[]>;
-export declare function orderedBm25Sql(field: 'title' | 'summary' | 'text', termCount: number): string;
+import type { DuckdbFtsBuilders } from './engine.js';
+export declare function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]>;
+export declare function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string;

package/dist/cjs/store/duckdb/ordered-bm25.d.ts

@@ -1,4 +1,4 @@
 import type { Connection } from '../types.js';
-export declare const FTS_STEMMER = "porter";
-export declare function nativeQueryTerms(conn: Connection, query: string): Promise<string[]>;
-export declare function orderedBm25Sql(field: 'title' | 'summary' | 'text', termCount: number): string;
+import type { DuckdbFtsBuilders } from './engine.js';
+export declare function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]>;
+export declare function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string;

package/dist/cjs/store/duckdb/ordered-bm25.js

@@ -9,9 +9,6 @@ function _export(target, all) {
     });
 }
 _export(exports, {
-    get FTS_STEMMER () {
-        return FTS_STEMMER;
-    },
     get nativeQueryTerms () {
         return nativeQueryTerms;
     },
@@ -144,8 +141,7 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
-var FTS_STEMMER = 'porter';
-function nativeQueryTerms(conn, query) {
+function nativeQueryTerms(conn, fts, query) {
     return _async_to_generator(function() {
         var stmt, rows;
         return _ts_generator(this, function(_state) {
@@ -153,7 +149,7 @@ function nativeQueryTerms(conn, query) {
                 case 0:
                     return [
                         4,
-                        conn.prepare("\n    WITH tokens AS (SELECT unnest(fts_main_content.tokenize(?)) AS raw_term)\n    SELECT min(raw_term) AS term FROM tokens\n    WHERE raw_term <> '' GROUP BY stem(raw_term, '".concat(FTS_STEMMER, "') ORDER BY term\n  "))
+                        conn.prepare(fts.queryTerms())
                     ];
                 case 1:
                     stmt = _state.sent();
@@ -174,12 +170,12 @@ function nativeQueryTerms(conn, query) {
         });
     })();
 }
-function orderedBm25Sql(field, termCount) {
+function orderedBm25Sql(fts, field, termCount) {
     if (termCount === 0) return 'NULL';
     var contributions = Array.from({
         length: termCount
     }, function() {
-        return 'SELECT fts_main_content.match_bm25(content."path", ?, fields := \''.concat(field, "', conjunctive := false) AS score");
+        return fts.termScore(field);
     });
     return "(SELECT sum(score ORDER BY score) FROM (".concat(contributions.join(' UNION ALL '), "))");
 }

package/dist/cjs/store/duckdb/ordered-bm25.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import type { Connection } from '../types.ts';\n\nexport const FTS_STEMMER = 'porter';\n\n// Keep deduplication native, but return original tokens so native scoring stems once.\nexport async function nativeQueryTerms(conn: Connection, query: string): Promise<string[]> {\n  const stmt = await conn.prepare(`\n    WITH tokens AS (SELECT unnest(fts_main_content.tokenize(?)) AS raw_term)\n    SELECT min(raw_term) AS term FROM tokens\n    WHERE raw_term <> '' GROUP BY stem(raw_term, '${FTS_STEMMER}') ORDER BY term\n  `);\n  const rows = (await stmt.all(query)) as Array<{ term: string }>;\n  return rows.map(({ term }) => term);\n}\n\n// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.\n// Only the number of terms shapes this authored SQL; query values are always bound.\nexport function orderedBm25Sql(field: 'title' | 'summary' | 'text', termCount: number): string {\n  if (termCount === 0) return 'NULL';\n  const contributions = Array.from({ length: termCount }, () => `SELECT fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false) AS score`);\n  return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;\n}\n"],"names":["FTS_STEMMER","nativeQueryTerms","orderedBm25Sql","conn","query","stmt","rows","prepare","all","map","term","field","termCount","contributions","Array","from","length","join"],"mappings":";;;;;;;;;;;QAEaA;eAAAA;;QAGSC;eAAAA;;QAYNC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAfT,IAAMF,cAAc;AAGpB,SAAeC,iBAAiBE,IAAgB,EAAEC,KAAa;;YAC9DC,MAKAC;;;;oBALO;;wBAAMH,KAAKI,OAAO,CAAC,AAAC,mLAG6B,OAAZP,aAAY;;;oBAHxDK,OAAO;oBAKC;;wBAAMA,KAAKG,GAAG,CAACJ;;;oBAAvBE,OAAQ;oBACd;;wBAAOA,KAAKG,GAAG,CAAC;gCAAGC,aAAAA;mCAAWA;;;;;IAChC;;AAIO,SAASR,eAAeS,KAAmC,EAAEC,SAAiB;IACnF,IAAIA,cAAc,GAAG,OAAO;IAC5B,IAAMC,gBAAgBC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAU,GAAG;eAAM,AAAC,qEAAyE,OAAND,OAAM;;IACxI,OAAO,AAAC,2CAA4E,OAAlCE,cAAcI,IAAI,CAAC,gBAAe;AACtF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import type { Connection } from '../types.ts';\nimport type { DuckdbFtsBuilders } from './engine.ts';\n\n// Keep deduplication native, but return original tokens so native scoring stems once.\nexport async function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]> {\n  const stmt = await conn.prepare(fts.queryTerms());\n  const rows = (await stmt.all(query)) as Array<{ term: string }>;\n  return rows.map(({ term }) => term);\n}\n\n// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.\n// Only the number of terms shapes this authored SQL; query values are always bound.\nexport function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string {\n  if (termCount === 0) return 'NULL';\n  const contributions = Array.from({ length: termCount }, () => fts.termScore(field));\n  return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;\n}\n"],"names":["nativeQueryTerms","orderedBm25Sql","conn","fts","query","stmt","rows","prepare","queryTerms","all","map","term","field","termCount","contributions","Array","from","length","termScore","join"],"mappings":";;;;;;;;;;;QAIsBA;eAAAA;;QAQNC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AART,SAAeD,iBAAiBE,IAAgB,EAAEC,GAAsB,EAAEC,KAAa;;YACtFC,MACAC;;;;oBADO;;wBAAMJ,KAAKK,OAAO,CAACJ,IAAIK,UAAU;;;oBAAxCH,OAAO;oBACC;;wBAAMA,KAAKI,GAAG,CAACL;;;oBAAvBE,OAAQ;oBACd;;wBAAOA,KAAKI,GAAG,CAAC;gCAAGC,aAAAA;mCAAWA;;;;;IAChC;;AAIO,SAASV,eAAeE,GAAsB,EAAES,KAAmC,EAAEC,SAAiB;IAC3G,IAAIA,cAAc,GAAG,OAAO;IAC5B,IAAMC,gBAAgBC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAU,GAAG;eAAMV,IAAIe,SAAS,CAACN;;IAC5E,OAAO,AAAC,2CAA4E,OAAlCE,cAAcK,IAAI,CAAC,gBAAe;AACtF"}

package/dist/cjs/store/engine.d.cts

@@ -0,0 +1,28 @@
+import { SenseError } from '../errors.js';
+export declare const ENGINE_IDENTITY_META_KEY = "engine_identity";
+export declare function engineIdentity(pkg: string, version: string | undefined): string;
+/**
+ * One version-specific replacement of a store's SQL builders. `range` matches the engine version (the npm package's,
+ * or Node's bundled SQLite for sqlite), prereleases included since every `@duckdb/node-api` release is one;
+ * `reason` names the observed difference.
+ */
+export interface EnginePatch<Builders> {
+    range: string;
+    reason: string;
+    builders: Partial<Builders>;
+}
+export declare function selectBuilders<Builders extends object>(defaults: Builders, patches: readonly EnginePatch<Builders>[], version: string | undefined): Builders;
+export interface EngineCheckDoc {
+    path: string;
+    title: string;
+    summary: string;
+    text: string;
+}
+export declare const ENGINE_CHECK_DOCS: readonly EngineCheckDoc[];
+export declare const ENGINE_CHECK_QUERIES: ReadonlyArray<{
+    terms: string;
+    paths: string[];
+}>;
+export declare function npmRemedy(pkg: string, range: string): string;
+export declare function checkEngine(store: string, identity: string, remedy: string, query: (terms: string) => Promise<string[]>): Promise<void>;
+export declare function engineCheckFailed(store: string, identity: string, remedy: string, failure: string): SenseError;

package/dist/cjs/store/engine.d.ts

@@ -0,0 +1,28 @@
+import { SenseError } from '../errors.js';
+export declare const ENGINE_IDENTITY_META_KEY = "engine_identity";
+export declare function engineIdentity(pkg: string, version: string | undefined): string;
+/**
+ * One version-specific replacement of a store's SQL builders. `range` matches the engine version (the npm package's,
+ * or Node's bundled SQLite for sqlite), prereleases included since every `@duckdb/node-api` release is one;
+ * `reason` names the observed difference.
+ */
+export interface EnginePatch<Builders> {
+    range: string;
+    reason: string;
+    builders: Partial<Builders>;
+}
+export declare function selectBuilders<Builders extends object>(defaults: Builders, patches: readonly EnginePatch<Builders>[], version: string | undefined): Builders;
+export interface EngineCheckDoc {
+    path: string;
+    title: string;
+    summary: string;
+    text: string;
+}
+export declare const ENGINE_CHECK_DOCS: readonly EngineCheckDoc[];
+export declare const ENGINE_CHECK_QUERIES: ReadonlyArray<{
+    terms: string;
+    paths: string[];
+}>;
+export declare function npmRemedy(pkg: string, range: string): string;
+export declare function checkEngine(store: string, identity: string, remedy: string, query: (terms: string) => Promise<string[]>): Promise<void>;
+export declare function engineCheckFailed(store: string, identity: string, remedy: string, failure: string): SenseError;