sensemaking 0.19.1 → 0.20.0

package/dist/cjs/store/duckdb/open.js

@@ -15,18 +15,18 @@ _export(exports, {
     get SCHEMA_VERSION () {
         return SCHEMA_VERSION;
     },
+    get duckdbOpenDialect () {
+        return duckdbOpenDialect;
+    },
     get openDuckdb () {
         return openDuckdb;
     }
 });
-var _nodefs = require("node:fs");
-var _nodepath = require("node:path");
 var _indexts = require("../../config/index.js");
-var _handoffts = require("../../embed/handoff.js");
 var _typests = require("../../embed/types.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
-var _cachets = require("../cache.js");
+var _opents = require("../open.js");
 var _sharedts = require("../shared.js");
 var _connectionts = require("./connection.js");
 var _nativets = require("./native.js");
@@ -165,10 +165,10 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.duckdb';
-var SCHEMA_VERSION = '2';
+var SCHEMA_VERSION = '3';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
 // No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.
-function ensureSchema(conn, cfg) {
+function ensureSchema(_handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
         return _ts_generator(this, function(_state) {
@@ -311,45 +311,50 @@ function ensureSchema(conn, cfg) {
         });
     })();
 }
-function connect(cfg) {
+// Order matters: the connection must be gone before the instance closes the WAL.
+function close(handle) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            handle.duckdb.disconnectSync();
+            handle.instance.closeSync();
+            return [
+                2
+            ];
+        });
+    })();
+}
+function connect(dbPath, _cfg) {
     return _async_to_generator(function() {
-        var stateDir, dbPath, DuckDBInstance, err, duckdb, instance, err1, closed, conn, version, features, wantFeatures, reason, _ref, parsed, warnings, err2;
+        var DuckDBInstance, err, duckdb, instance, err1, conn, err2;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
-                    (0, _nodefs.mkdirSync)(stateDir, {
-                        recursive: true
-                    });
-                    dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
-                    _state.label = 1;
-                case 1:
                     _state.trys.push([
-                        1,
-                        3,
+                        0,
+                        2,
                         ,
-                        4
+                        3
                     ]);
                     return [
                         4,
                         (0, _nativets.duckdbApi)()
                     ];
-                case 2:
+                case 1:
                     DuckDBInstance = _state.sent().DuckDBInstance;
                     return [
                         3,
-                        4
+                        3
                     ];
-                case 3:
+                case 2:
                     err = _state.sent();
                     if (_instanceof(err, _errorsts.SenseError)) throw err;
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" needs the '.concat(_nativets.DUCKDB_PACKAGE, " package (").concat(err.message, ")"));
-                case 4:
+                case 3:
                     _state.trys.push([
-                        4,
-                        7,
+                        3,
+                        6,
                         ,
-                        8
+                        7
                     ]);
                     return [
                         4,
@@ -357,7 +362,7 @@ function connect(cfg) {
                             storage_compatibility_version: 'v1.5.0'
                         })
                     ];
-                case 5:
+                case 4:
                     // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns ("VARIANT
                     // columns are not supported in storage versions prior to v1.5.0"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.
                     instance = _state.sent();
@@ -365,102 +370,54 @@ function connect(cfg) {
                         4,
                         instance.connect()
                     ];
-                case 6:
+                case 5:
                     duckdb = _state.sent();
                     return [
                         3,
-                        8
+                        7
                     ];
-                case 7:
+                case 6:
                     err1 = _state.sent();
                     // create() may have succeeded before connect() failed: close it, or its WAL stays open.
                     instance === null || instance === void 0 ? void 0 : instance.closeSync();
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "duckdb" failed to open '.concat(dbPath, ": ").concat(err1.message));
-                case 8:
-                    // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL
-                    // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.
-                    closed = false;
-                    _state.label = 9;
-                case 9:
+                case 7:
                     _state.trys.push([
+                        7,
                         9,
-                        16,
                         ,
-                        17
+                        11
                     ]);
                     return [
                         4,
                         (0, _sqlfunctionsts.registerFunctions)(duckdb)
                     ];
-                case 10:
+                case 8:
                     _state.sent();
                     conn = (0, _connectionts.createConnection)(duckdb);
-                    return [
-                        4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)')
-                    ];
-                case 11:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'schema_version')
-                    ];
-                case 12:
-                    version = _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'features')
-                    ];
-                case 13:
-                    features = _state.sent();
-                    wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
-                    if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-                        // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
-                        // rebuild instead, same as sqlite's open() (see its comment for why this is announced).
-                        reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
-                        console.error("sense: ".concat(reason, "; rebuilding the index"));
-                        // Close before clearCache deletes the files underneath the still-open instance.
-                        closed = true;
-                        duckdb.disconnectSync();
-                        instance.closeSync();
-                        (0, _cachets.clearCache)(cfg);
-                        return [
-                            2,
-                            connect(cfg)
-                        ];
-                    }
-                    return [
-                        4,
-                        ensureSchema(conn, cfg)
-                    ];
-                case 14:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir)
-                    ];
-                case 15:
-                    _ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
                     return [
                         2,
                         {
-                            instance: instance,
-                            duckdb: duckdb,
-                            conn: conn,
-                            cfg: cfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
+                            handle: {
+                                instance: instance,
+                                duckdb: duckdb
+                            },
+                            conn: conn
                         }
                     ];
-                case 16:
+                case 9:
                     err2 = _state.sent();
-                    if (!closed) {
-                        duckdb.disconnectSync();
-                        instance === null || instance === void 0 ? void 0 : instance.closeSync();
-                    }
+                    return [
+                        4,
+                        close({
+                            instance: instance,
+                            duckdb: duckdb
+                        })
+                    ];
+                case 10:
+                    _state.sent();
                     throw err2;
-                case 17:
+                case 11:
                     return [
                         2
                     ];
@@ -468,32 +425,24 @@ function connect(cfg) {
         });
     })();
 }
+var duckdbOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: _reconcilets.duckdbDialect,
+    connect: connect,
+    close: close,
+    ensureSchema: ensureSchema,
+    createStore: function createStore(handle, conn) {
+        return (0, _storets.createStore)(handle.instance, handle.duckdb, conn);
+    }
+};
 function openDuckdb(cfg) {
     return _async_to_generator(function() {
-        var _ref, instance, duckdb, conn, resolvedCfg, dbPath, parsed, warnings, store;
         return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    return [
-                        4,
-                        connect(cfg)
-                    ];
-                case 1:
-                    _ref = _state.sent(), instance = _ref.instance, duckdb = _ref.duckdb, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
-                    store = (0, _storets.createStore)(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);
-                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-                    (0, _handoffts.rekeyChunkText)(conn, store);
-                    return [
-                        2,
-                        {
-                            store: store,
-                            cfg: resolvedCfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
-                        }
-                    ];
-            }
+            return [
+                2,
+                (0, _opents.openWithDialect)(cfg, duckdbOpenDialect)
+            ];
         });
     })();
 }

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openDuckdb","ensureSchema","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","DuckDBInstance","err","duckdb","instance","closed","version","features","wantFeatures","reason","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","closeSync","registerFunctions","createConnection","console","error","disconnectSync","clearCache","reconcile","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAkBaA;eAAAA;;QAGAC;eAAAA;;QA+FSC;eAAAA;;;sBApHI;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;wBACM;4BAEA;wBACS;2BAChB;8BACQ;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAU9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAKlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMW,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQX,GAAmB;;YAClCY,UAEAC,QAIFC,gBAGKC,KAKLC,QACAC,UAMKF,MAOLG,QAGInB,MAIAoB,SACAC,UACAC,cAIEC,QAYqB,MAArBC,QAAQC,UAETT;;;;oBAvDHH,WAAWa,IAAAA,cAAI,EAACzB,IAAI0B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUjB;;;;;;;;;oBAMN;;wBAAMmC,IAAAA,mBAAS;;;oBAAlChB,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAMnB,eAAeoB,MAAM,CAACrB,QAAQ;4BAAEsB,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KlB,WAAW;oBACF;;wBAAMA,SAASN,OAAO;;;oBAA/BK,SAAS;;;;;;oBACFD;oBACP,wFAAwF;oBACxFE,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACnB,MAAM,IAAIL,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAErH,2FAA2F;oBAC3F,uFAAuF;oBACnFf,SAAS;;;;;;;;;oBAEX;;wBAAMmB,IAAAA,iCAAiB,EAACrB;;;oBAAxB;oBACMjB,OAAOuC,IAAAA,8BAAgB,EAACtB;oBAE9B;;wBAAMjB,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACR,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMZ,IAAAA,iBAAO,EAACR,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeZ,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;oBACnD,IAAI,AAACS,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;wBACxG,4FAA4F;wBAC5F,wFAAwF;wBAClFC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;wBACnH2C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPlB,QAAO;wBAC/B,gFAAgF;wBAChFJ,SAAS;wBACTF,OAAOyB,cAAc;wBACrBxB,SAASmB,SAAS;wBAClBM,IAAAA,mBAAU,EAAC1C;wBACX;;4BAAOW,QAAQX;;oBACjB;oBAEA;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAE6B;;wBAAM2C,IAAAA,sBAAS,EAAC5C,MAAMC,KAAKA,IAAI0B,OAAO;;;oBAAtC,OAAA,eAArBH,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAChB;;wBAAO;4BAAEP,UAAAA;4BAAUD,QAAAA;4BAAQjB,MAAAA;4BAAMC,KAAAA;4BAAKa,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBACxDT;oBACP,IAAI,CAACG,QAAQ;wBACXF,OAAOyB,cAAc;wBACrBxB,qBAAAA,+BAAAA,SAAUmB,SAAS;oBACrB;oBACA,MAAMrB;;;;;;;IAEV;;AAEO,SAAelB,WAAWG,GAAmB;;YAC6B,MAAvEiB,UAAUD,QAAQjB,MAAW6C,aAAa/B,QAAQU,QAAQC,UAC5DqB;;;;oBADyE;;wBAAMlC,QAAQX;;;oBAAd,OAAA,eAAvEiB,WAAuE,KAAvEA,UAAUD,SAA6D,KAA7DA,QAAQjB,OAAqD,KAArDA,MAAW6C,cAA0C,KAA/C5C,KAAkBa,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC5DqB,QAAQC,IAAAA,oBAAW,EAAC7B,UAAUD,QAAQjB,MAAM6C,aAAaA,YAAYlB,OAAO;oBAClF,0FAA0F;oBAC1FqB,IAAAA,yBAAc,EAAChD,MAAM8C;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO7C,KAAK4C;4BAAa/B,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAUSC;eAAAA;;;uBAnGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACAd,cAAAA;IACAqC,aAAa,SAAbA,YAActB,QAAQb;eAASmC,IAAAA,oBAAW,EAACtB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOmC,IAAAA,uBAAe,EAACnC,KAAKL;;;IAC9B"}

package/dist/cjs/store/duckdb/reconcile.d.cts

@@ -1,6 +1,2 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
+import type { ReconcileDialect } from '../types.js';
+export declare const duckdbDialect: ReconcileDialect;

package/dist/cjs/store/duckdb/reconcile.d.ts

@@ -1,6 +1,2 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
+import type { ReconcileDialect } from '../types.js';
+export declare const duckdbDialect: ReconcileDialect;