sensemaking 0.19.2 → 0.20.0

package/dist/cjs/store/sqlite/open.js

@@ -17,27 +17,23 @@ _export(exports, {
     get SCHEMA_VERSION () {
         return SCHEMA_VERSION;
     },
-    get docCount () {
-        return docCount;
-    },
     get openSqlite () {
         return openSqlite;
+    },
+    get sqliteOpenDialect () {
+        return sqliteOpenDialect;
     }
 });
-var _nodefs = require("node:fs");
-var _nodepath = require("node:path");
 var _nodesqlite = require("node:sqlite");
 var _indexts = require("../../config/index.js");
-var _handoffts = require("../../embed/handoff.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
-var _cachets = require("../cache.js");
-var _reconcilets = require("../reconcile.js");
+var _indexts2 = require("../../scan/index.js");
+var _opents = require("../open.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _connectionts = require("./connection.js");
-var _reconcilets1 = require("./reconcile.js");
-var _signaturets = require("./signature.js");
+var _reconcilets = require("./reconcile.js");
 var _sqlfunctionsts = require("./sql-functions.js");
 var _storets = require("./store.js");
 function _array_like_to_array(arr, len) {
@@ -193,7 +189,7 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 var DB_FILENAME = 'cache.db';
-var SCHEMA_VERSION = '18';
+var SCHEMA_VERSION = '19';
 // unicode61 splits on spaces, so a language written without them indexes a whole run as one
 // token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.
 var DEFAULT_TOKENIZE = 'porter unicode61';
@@ -242,7 +238,7 @@ function createContentTable(conn, tokenize) {
 }
 // Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
 // can't dump file text into context. Features add their own tables after the core ones.
-function ensureSchema(conn, cfg, tokenize) {
+function ensureSchemaTables(conn, cfg, tokenize) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
         return _ts_generator(this, function(_state) {
@@ -388,75 +384,65 @@ function setJournalWal(db) {
         }
     }
 }
-function connect(cfg) {
+function connect(dbPath, cfg) {
+    return _async_to_generator(function() {
+        var db, conn, tokenize;
+        return _ts_generator(this, function(_state) {
+            db = new _nodesqlite.DatabaseSync(dbPath);
+            try {
+                // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive
+                // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.
+                db.exec('PRAGMA busy_timeout = 30000');
+                // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy
+                // handler for it, so a concurrent cold open needs its own bounded wait.
+                setJournalWal(db);
+                (0, _sqlfunctionsts.registerFunctions)(db, (0, _indexts.contentTokenize)(cfg) === undefined);
+                conn = (0, _connectionts.createConnection)(db);
+                // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd
+                // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.
+                tokenize = resolveTokenize(db, cfg);
+                return [
+                    2,
+                    {
+                        handle: {
+                            db: db,
+                            tokenize: tokenize
+                        },
+                        conn: conn
+                    }
+                ];
+            } catch (err) {
+                // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and
+                // scratch cleanup fails with EPERM/EBUSY.
+                db.close();
+                throw err;
+            }
+            return [
+                2
+            ];
+        });
+    })();
+}
+function close(handle) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            handle.db.close();
+            return [
+                2
+            ];
+        });
+    })();
+}
+// Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived
+// and tokenizer-independent. Any other signature change takes the full clear/reopen in open.ts.
+function partialRebuild(handle, conn, _cfg, changedKeys) {
     return _async_to_generator(function() {
-        var stateDir, dbPath, db, closed, _ref, conn, tokenize, version, features, wantFeatures, tokenizeOnlyRebuild, changedKeys, changed, stored, rebuildWarnings, recordedMaxMs, _ref1, parsed, warnings, err;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
-                    (0, _nodefs.mkdirSync)(stateDir, {
-                        recursive: true
-                    });
-                    dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
-                    db = new _nodesqlite.DatabaseSync(dbPath);
-                    // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and
-                    // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.
-                    closed = false;
-                    _state.label = 1;
-                case 1:
-                    _state.trys.push([
-                        1,
-                        15,
-                        ,
-                        16
-                    ]);
-                    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive
-                    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.
-                    db.exec('PRAGMA busy_timeout = 30000');
-                    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy
-                    // handler for it, so a concurrent cold open needs its own bounded wait.
-                    setJournalWal(db);
-                    (0, _sqlfunctionsts.registerFunctions)(db, (0, _indexts.contentTokenize)(cfg) === undefined);
-                    conn = (0, _connectionts.createConnection)(db);
-                    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
-                    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd
-                    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.
-                    tokenize = resolveTokenize(db, cfg);
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'schema_version')
-                    ];
-                case 2:
-                    version = _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'features')
-                    ];
-                case 3:
-                    features = _state.sent();
-                    wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
-                    tokenizeOnlyRebuild = false;
-                    if (!(version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures)) return [
-                        3,
-                        8
-                    ];
-                    // Indexing derives from presets, so a config edit rebuilding the cache must say so and
-                    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
-                    if (version !== null && version !== SCHEMA_VERSION) {
-                        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
-                        closed = true;
-                        db.close();
-                        (0, _cachets.clearCache)(cfg);
-                        return [
-                            2,
-                            connect(cfg)
-                        ];
-                    }
-                    changedKeys = (0, _signaturets.changedSignatureKeys)(features !== null && features !== void 0 ? features : '', wantFeatures);
                     if (!(changedKeys.size === 1 && changedKeys.has('tokenize'))) return [
-                        3,
-                        5
+                        2,
+                        false
                     ];
                     console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
                     // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather
@@ -476,7 +462,7 @@ function connect(cfg) {
                                             _state.sent();
                                             return [
                                                 4,
-                                                createContentTable(conn, tokenize)
+                                                createContentTable(conn, handle.tokenize)
                                             ];
                                         case 2:
                                             _state.sent();
@@ -488,173 +474,175 @@ function connect(cfg) {
                             })();
                         }, _transactionts.BEGIN_WRITE)
                     ];
-                case 4:
+                case 1:
                     _state.sent();
-                    tokenizeOnlyRebuild = true;
                     return [
-                        3,
-                        8
-                    ];
-                case 5:
-                    if (!(changedKeys.size === 1 && changedKeys.has('embed') && (0, _signaturets.embedIdentityAdopted)(features !== null && features !== void 0 ? features : '', wantFeatures))) return [
-                        3,
-                        7
+                        2,
+                        true
                     ];
-                    // First sight of a resolved weight identity: the model itself hasn't changed, so
-                    // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.
-                    console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
+            }
+        });
+    })();
+}
+// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
+// and no feature extractors run. Warnings come from the reparse, since mtimes stay untouched and reconcile would never reparse them.
+function postSchemaRebuild(_handle, conn, cfg, baseDir, wantFeatures) {
+    return _async_to_generator(function() {
+        var stmt, known, _, files, segmenting, warnings, rows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, _warnings, _parseFile, doc, fileWarnings;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
                     return [
                         4,
-                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
-                    ];
-                case 6:
-                    _state.sent();
-                    return [
-                        3,
-                        8
+                        conn.prepare('SELECT "path" FROM frontmatter')
                     ];
-                case 7:
-                    changed = (0, _signaturets.signatureDiff)(features !== null && features !== void 0 ? features : '', wantFeatures);
-                    console.error("sense: config change (".concat(changed, ") rebuilds the index"));
-                    closed = true;
-                    db.close();
-                    (0, _cachets.clearCache)(cfg);
+                case 1:
+                    stmt = _state.sent();
+                    _ = Set.bind;
                     return [
-                        2,
-                        connect(cfg)
+                        4,
+                        stmt.all()
                     ];
-                case 8:
-                    // Meta can lie after a crash between table creation and the signature write; the table's own
-                    // DDL cannot, so a mismatch here rebuilds whatever meta says.
-                    stored = storedTokenize(db);
-                    if (stored !== null && stored !== tokenize) {
-                        console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
-                        closed = true;
-                        db.close();
-                        (0, _cachets.clearCache)(cfg);
-                        return [
-                            2,
-                            connect(cfg)
-                        ];
+                case 2:
+                    known = new (_.apply(Set, [
+                        void 0,
+                        _state.sent().map(function(r) {
+                            return r.path;
+                        })
+                    ]));
+                    files = (0, _indexts2.listFiles)(cfg, baseDir).filter(function(f) {
+                        return known.has(f.relPath);
+                    });
+                    segmenting = (0, _indexts.contentTokenize)(cfg) === undefined;
+                    warnings = [];
+                    rows = [];
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = files[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            file = _step.value;
+                            ;
+                            _parseFile = (0, _indexts2.parseFile)(file), doc = _parseFile.doc, fileWarnings = _parseFile.warnings;
+                            (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
+                            rows.push((0, _reconcilets.contentRow)(doc, segmenting));
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
                     }
-                    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
-                    // here both see it missing and the second ALTER fails with a duplicate column.
                     return [
                         4,
                         (0, _transactionts.withTransaction)(conn, function() {
-                            return ensureSchema(conn, cfg, tokenize);
+                            return _async_to_generator(function() {
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            if (!(rows.length > 0)) return [
+                                                3,
+                                                2
+                                            ];
+                                            return [
+                                                4,
+                                                conn.runBatch(_reconcilets.INSERT_CONTENT_SQL, rows)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            _state.label = 2;
+                                        case 2:
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
                         }, _transactionts.BEGIN_WRITE)
                     ];
-                case 9:
+                case 3:
                     _state.sent();
-                    rebuildWarnings = [];
-                    if (!tokenizeOnlyRebuild) return [
-                        3,
-                        12
-                    ];
-                    return [
-                        4,
-                        (0, _signaturets.rebuildContentTable)(conn, cfg, cfg.baseDir)
-                    ];
-                case 10:
-                    rebuildWarnings = _state.sent();
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
                     ];
-                case 11:
+                case 4:
                     _state.sent();
-                    _state.label = 12;
-                case 12:
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
-                    ];
-                case 13:
-                    recordedMaxMs = Number.apply(void 0, [
-                        (_ref = _state.sent()) !== null && _ref !== void 0 ? _ref : '0'
-                    ]);
-                    db.exec("PRAGMA busy_timeout = ".concat(Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)));
-                    return [
-                        4,
-                        (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir, _reconcilets1.sqliteDialect)
-                    ];
-                case 14:
-                    _ref1 = _state.sent(), parsed = _ref1.parsed, warnings = _ref1.warnings;
                     return [
                         2,
-                        {
-                            db: db,
-                            conn: conn,
-                            cfg: cfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: _to_consumable_array(rebuildWarnings).concat(_to_consumable_array(warnings))
-                        }
-                    ];
-                case 15:
-                    err = _state.sent();
-                    if (!closed) db.close();
-                    throw err;
-                case 16:
-                    return [
-                        2
+                        warnings
                     ];
             }
         });
     })();
 }
-function openSqlite(cfg) {
+// Meta can lie after a crash between table creation and the signature write; the table's own
+// DDL cannot, so a mismatch here rebuilds whatever meta says.
+function extraRebuildReason(handle) {
+    var stored = storedTokenize(handle.db);
+    if (stored !== null && stored !== handle.tokenize) return 'cache was built with a different content tokenizer';
+    return null;
+}
+function ensureSchema(handle, conn, cfg) {
     return _async_to_generator(function() {
-        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings, store;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
+                    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
+                    // here both see it missing and the second ALTER fails with a duplicate column.
                     return [
                         4,
-                        connect(cfg)
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return ensureSchemaTables(conn, cfg, handle.tokenize);
+                        }, _transactionts.BEGIN_WRITE)
                     ];
                 case 1:
-                    _ref = _state.sent(), db = _ref.db, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
-                    store = (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir);
-                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-                    (0, _handoffts.rekeyChunkText)(conn, store);
+                    _state.sent();
                     return [
-                        2,
-                        {
-                            store: store,
-                            cfg: resolvedCfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
-                        }
+                        2
                     ];
             }
         });
     })();
 }
-function docCount(store) {
+function setDerivedBusyTimeout(handle, _conn, ms) {
     return _async_to_generator(function() {
-        var stmt;
         return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    return [
-                        4,
-                        store.prepare('SELECT COUNT(*) AS n FROM frontmatter')
-                    ];
-                case 1:
-                    stmt = _state.sent();
-                    return [
-                        4,
-                        stmt.get()
-                    ];
-                case 2:
-                    return [
-                        2,
-                        _state.sent().n
-                    ];
-            }
+            handle.db.exec("PRAGMA busy_timeout = ".concat(ms));
+            return [
+                2
+            ];
+        });
+    })();
+}
+var sqliteOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: _reconcilets.sqliteDialect,
+    connect: connect,
+    close: close,
+    partialRebuild: partialRebuild,
+    postSchemaRebuild: postSchemaRebuild,
+    extraRebuildReason: extraRebuildReason,
+    ensureSchema: ensureSchema,
+    setDerivedBusyTimeout: setDerivedBusyTimeout,
+    createStore: function createStore(handle, conn, cfg) {
+        return (0, _storets.createStore)(handle.db, conn, cfg);
+    }
+};
+function openSqlite(cfg) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            return [
+                2,
+                (0, _opents.openWithDialect)(cfg, sqliteOpenDialect)
+            ];
         });
     })();
 }

package/dist/cjs/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, signatureDiff } from './signature.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and\n  // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.\n  let closed = false;\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n      // and tokenizer-independent. Any other signature change takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n        // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n        await withTransaction(\n          conn,\n          async () => {\n            await conn.exec('DROP TABLE IF EXISTS content');\n            await createContentTable(conn, tokenize);\n          },\n          BEGIN_WRITE\n        );\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's own\n    // DDL cannot, so a mismatch here rebuilds whatever meta says.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n    // here both see it missing and the second ALTER fails with a duplicate column.\n    await withTransaction(conn, () => ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, sqliteDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","docCount","openSqlite","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","contentTokenize","undefined","literal","replace","exec","err","SenseError","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","setJournalWal","deadline","Date","now","test","Atomics","wait","Int32Array","SharedArrayBuffer","connect","stateDir","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","changedKeys","changed","stored","rebuildWarnings","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","DatabaseSync","registerFunctions","createConnection","console","error","close","clearCache","changedSignatureKeys","size","has","withTransaction","BEGIN_WRITE","embedIdentityAdopted","signatureDiff","rebuildContentTable","Number","Math","min","max","reconcile","sqliteDialect","resolvedCfg","store","createStore","rekeyChunkText","stmt","n"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAoBpFA;eAAAA;;QAGAC;eAAAA;;QA0MSC;eAAAA;;QARAC;eAAAA;;;sBAxNI;wBACL;0BACQ;uBAEgC;yBAC9B;wBACJ;wBACc;uBACd;2BACD;wBACO;6BACY;4BAEZ;4BACH;2BACiE;8BAC7D;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAmB9B,4FAA4F;AAC5F,4FAA4F;AAC5F,IAAMG,mBAAmB;AAEzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,IAAMC,aAAaC,IAAAA,wBAAe,EAACF;IACnC,IAAIC,eAAeE,WAAW,OAAON;IACrC,IAAMO,UAAUH,WAAWI,OAAO,CAAC,MAAM;IACzC,IAAI;QACFN,GAAGO,IAAI,CAAC;QACRP,GAAGO,IAAI,CAAC,AAAC,4EAAmF,OAARF,SAAQ;QAC5FL,GAAGO,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIC,oBAAU,CAAC,kBAAkB,AAAC,qBAA2E,OAAvDP,YAAW,8CAAmE,OAAvB,AAACM,IAAcE,OAAO,EAAC;IAC5I;IACA,OAAOL;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASM,eAAeX,EAAgB;IACtC,IAAMY,MAAMZ,GAAGa,OAAO,CAAC,wDAAwDC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,IAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,SAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;;;;;oBAClE;;wBAAMD,KAAKZ,IAAI,CAAC,AAAC,6IAAqJ,OAATa,UAAS;;;oBAAtK;;;;;;IACF;;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeC,aAAaF,IAAgB,EAAElB,GAAW,EAAEmB,QAAgB;;YASpE,2BAAA,mBAAA,gBAAA,WAAA,OAAME;;;;oBARX;;wBAAMH,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA,wFAAwF;oBACxF,4FAA4F;oBAC5F;;wBAAMW,mBAAmBC,MAAMC;;;oBAA/B;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMY,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBgB,IAAAA,wBAAc,EAACtB;;;2BAAhC,6BAAA,QAAA;;;;oBAAMqB,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACL;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,kBAAkBxB;;;oBAAtC;;;oBACjD;;wBAAM8B,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,YAAYQ,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASC,cAAc7B,EAAgB;IACrC,IAAM8B,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFhC,GAAGO,IAAI,CAAC;YACR;QACF,EAAE,OAAOC,KAAK;YACZ,IAAIuB,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BG,IAAI,CAAC,AAACzB,IAAcE,OAAO,GAAG,MAAMF;YAC9F0B,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,SAAeC,QAAQrC,GAAmB;;YAClCsC,UAEAC,QAEAxC,IAGFyC,QAuF4B,MA9ExBtB,MAMAC,UAIAsB,SACAC,UACAC,cACFC,qBAWIC,aAsBEC,SAWJC,QAaFC,iBAQEC,eAGuB,OAArBC,QAAQC,UAGT5C;;;;oBApGH+B,WAAWc,IAAAA,cAAI,EAACpD,IAAIqD,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACjB,UAAU;wBAAEkB,WAAW;oBAAK;oBAChCjB,SAASa,IAAAA,cAAI,EAACd,UAAU7C;oBAExBM,KAAK,IAAI0D,wBAAY,CAAClB;oBAC5B,6FAA6F;oBAC7F,wFAAwF;oBACpFC,SAAS;;;;;;;;;oBAEX,6FAA6F;oBAC7F,yFAAyF;oBACzFzC,GAAGO,IAAI,CAAC;oBACR,yFAAyF;oBACzF,wEAAwE;oBACxEsB,cAAc7B;oBACd2D,IAAAA,iCAAiB,EAAC3D,IAAIG,IAAAA,wBAAe,EAACF,SAASG;oBACzCe,OAAOyC,IAAAA,8BAAgB,EAAC5D;oBAE9BA,GAAGO,IAAI,CAAC;oBAER,2FAA2F;oBAC3F,8FAA8F;oBACxFa,WAAWrB,gBAAgBC,IAAIC;oBAIrB;;wBAAMwB,IAAAA,iBAAO,EAACN,MAAM;;;oBAA9BuB,UAAU;oBACC;;wBAAMjB,IAAAA,iBAAO,EAACN,MAAM;;;oBAA/BwB,WAAW;oBACXC,eAAejB,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ;oBAC/CiB,sBAAsB;yBACtB,CAAA,AAACH,YAAY,QAAQA,YAAY/C,kBAAoBgD,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,uFAAuF;oBACvF,wFAAwF;oBACxF,IAAIF,YAAY,QAAQA,YAAY/C,gBAAgB;wBAClDkE,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACTzC,GAAG+D,KAAK;wBACRC,IAAAA,mBAAU,EAAC/D;wBACX;;4BAAOqC,QAAQrC;;oBACjB;oBACM6C,cAAcmB,IAAAA,iCAAoB,EAACtB,qBAAAA,sBAAAA,WAAY,IAAIC;yBAGrDE,CAAAA,YAAYoB,IAAI,KAAK,KAAKpB,YAAYqB,GAAG,CAAC,WAAU,GAApDrB;;;;oBACFe,QAAQC,KAAK,CAAC;oBACd,2FAA2F;oBAC3F,yFAAyF;oBACzF;;wBAAMM,IAAAA,8BAAe,EACnBjD,MACA;;;;;4CACE;;gDAAMA,KAAKZ,IAAI,CAAC;;;4CAAhB;4CACA;;gDAAMW,mBAAmBC,MAAMC;;;4CAA/B;;;;;;4BACF;2BACAiD,0BAAW;;;oBANb;oBAQAxB,sBAAsB;;;;;;yBACbC,CAAAA,YAAYoB,IAAI,KAAK,KAAKpB,YAAYqB,GAAG,CAAC,YAAYG,IAAAA,iCAAoB,EAAC3B,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGE;;;;oBACT,iFAAiF;oBACjF,wFAAwF;oBACxFe,QAAQC,KAAK,CAAC;oBACd;;wBAAMpC,IAAAA,iBAAO,EAACP,MAAM,YAAYyB;;;oBAAhC;;;;;;oBAEMG,UAAUwB,IAAAA,0BAAa,EAAC5B,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9CiB,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARf,SAAQ;oBAC/CN,SAAS;oBACTzC,GAAG+D,KAAK;oBACRC,IAAAA,mBAAU,EAAC/D;oBACX;;wBAAOqC,QAAQrC;;;oBAInB,6FAA6F;oBAC7F,8DAA8D;oBACxD+C,SAASrC,eAAeX;oBAC9B,IAAIgD,WAAW,QAAQA,WAAW5B,UAAU;wBAC1CyC,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACTzC,GAAG+D,KAAK;wBACRC,IAAAA,mBAAU,EAAC/D;wBACX;;4BAAOqC,QAAQrC;;oBACjB;oBAEA,+FAA+F;oBAC/F,+EAA+E;oBAC/E;;wBAAMmE,IAAAA,8BAAe,EAACjD,MAAM;mCAAME,aAAaF,MAAMlB,KAAKmB;2BAAWiD,0BAAW;;;oBAAhF;oBAEIpB;yBACAJ,qBAAAA;;;;oBACgB;;wBAAM2B,IAAAA,gCAAmB,EAACrD,MAAMlB,KAAKA,IAAIqD,OAAO;;;oBAAlEL,kBAAkB;oBAClB;;wBAAMvB,IAAAA,iBAAO,EAACP,MAAM,YAAYyB;;;oBAAhC;;;oBAK4B;;wBAAMnB,IAAAA,iBAAO,EAACN,MAAM;;;oBAA5C+B,gBAAgBuB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1EzE,GAAGO,IAAI,CAAC,AAAC,yBAA8E,OAAtDmE,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAI1B,gBAAgB;oBAEjD;;wBAAM2B,IAAAA,sBAAS,EAAC1D,MAAMlB,KAAKA,IAAIqD,OAAO,EAAEwB,2BAAa;;;oBAArD,QAAA,eAArB3B,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAEpD,IAAAA;4BAAImB,MAAAA;4BAAMlB,KAAAA;4BAAKuC,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAU,AAAC,qBAAGH,wBAAiB,qBAAGG;wBAAU;;;oBAC7E5C;oBACP,IAAI,CAACiC,QAAQzC,GAAG+D,KAAK;oBACrB,MAAMvD;;;;;;;IAEV;;AAIO,SAAeX,WAAWI,GAAmB;;YACe,MAAzDD,IAAImB,MAAW4D,aAAavC,QAAQW,QAAQC,UAC9C4B;;;;oBAD2D;;wBAAM1C,QAAQrC;;;oBAAd,OAAA,eAAzDD,KAAyD,KAAzDA,IAAImB,OAAqD,KAArDA,MAAW4D,cAA0C,KAA/C9E,KAAkBuC,SAA6B,KAA7BA,QAAQW,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9C4B,QAAQC,IAAAA,oBAAW,EAACjF,IAAImB,MAAM4D,aAAaA,YAAYzB,OAAO;oBACpE,0FAA0F;oBAC1F4B,IAAAA,yBAAc,EAAC/D,MAAM6D;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO/E,KAAK8E;4BAAavC,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE;;AAEO,SAAexD,SAASoF,KAAY;;YACnCG;;;;oBAAO;;wBAAMH,MAAMnE,OAAO,CAAC;;;oBAA3BsE,OAAO;oBACJ;;wBAAMA,KAAKrE,GAAG;;;oBAAvB;;wBAAS,cAAoCsE,CAAC;;;;IAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { contentRow, INSERT_CONTENT_SQL, sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '19';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// The connection plus this store's own extra state (types.ts's OpenDialect<Handle>): the\n// resolved tokenizer, threaded through connect, the tokenize-only rebuild, and ensureSchema.\ninterface SqliteHandle {\n  db: DatabaseSync;\n  tokenize: string;\n}\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n    return { handle: { db, tokenize }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\n// Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n// and tokenizer-independent. Any other signature change takes the full clear/reopen in open.ts.\nasync function partialRebuild(handle: SqliteHandle, conn: Connection, _cfg: Config, changedKeys: Set<string>): Promise<boolean> {\n  if (!(changedKeys.size === 1 && changedKeys.has('tokenize'))) return false;\n  console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n  // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n  // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec('DROP TABLE IF EXISTS content');\n      await createContentTable(conn, handle.tokenize);\n    },\n    BEGIN_WRITE\n  );\n  return true;\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Warnings come from the reparse, since mtimes stay untouched and reconcile would never reparse them.\nasync function postSchemaRebuild(_handle: SqliteHandle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  await setMeta(conn, 'features', wantFeatures);\n  return warnings;\n}\n\n// Meta can lie after a crash between table creation and the signature write; the table's own\n// DDL cannot, so a mismatch here rebuilds whatever meta says.\nfunction extraRebuildReason(handle: SqliteHandle): string | null {\n  const stored = storedTokenize(handle.db);\n  if (stored !== null && stored !== handle.tokenize) return 'cache was built with a different content tokenizer';\n  return null;\n}\n\nasync function ensureSchema(handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg, handle.tokenize), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  partialRebuild,\n  postSchemaRebuild,\n  extraRebuildReason,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn, cfg) => createStore(handle.db, conn, cfg),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openSqlite","sqliteOpenDialect","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","contentTokenize","undefined","literal","replace","exec","err","SenseError","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchemaTables","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","setJournalWal","deadline","Date","now","test","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","DatabaseSync","registerFunctions","createConnection","handle","close","partialRebuild","_cfg","changedKeys","size","has","console","error","withTransaction","BEGIN_WRITE","postSchemaRebuild","_handle","baseDir","wantFeatures","stmt","known","files","segmenting","warnings","rows","file","parseFile","doc","fileWarnings","Set","all","map","r","path","listFiles","filter","f","relPath","push","contentRow","length","runBatch","INSERT_CONTENT_SQL","extraRebuildReason","stored","ensureSchema","setDerivedBusyTimeout","_conn","ms","filename","schemaVersion","reconcileDialect","sqliteDialect","createStore","openWithDialect"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAiBpFA;eAAAA;;QAGAC;eAAAA;;QAoLSC;eAAAA;;QAdTC;eAAAA;;;0BAzLgB;uBAEqB;wBACvB;wBACc;wBACJ;sBAEL;wBACC;6BACY;4BAEZ;2BAC6B;8BAC5B;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,4FAA4F;AAC5F,IAAMG,mBAAmB;AASzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,IAAMC,aAAaC,IAAAA,wBAAe,EAACF;IACnC,IAAIC,eAAeE,WAAW,OAAON;IACrC,IAAMO,UAAUH,WAAWI,OAAO,CAAC,MAAM;IACzC,IAAI;QACFN,GAAGO,IAAI,CAAC;QACRP,GAAGO,IAAI,CAAC,AAAC,4EAAmF,OAARF,SAAQ;QAC5FL,GAAGO,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIC,oBAAU,CAAC,kBAAkB,AAAC,qBAA2E,OAAvDP,YAAW,8CAAmE,OAAvB,AAACM,IAAcE,OAAO,EAAC;IAC5I;IACA,OAAOL;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASM,eAAeX,EAAgB;IACtC,IAAMY,MAAMZ,GAAGa,OAAO,CAAC,wDAAwDC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,IAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,SAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;;;;;oBAClE;;wBAAMD,KAAKZ,IAAI,CAAC,AAAC,6IAAqJ,OAATa,UAAS;;;oBAAtK;;;;;;IACF;;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeC,mBAAmBF,IAAgB,EAAElB,GAAW,EAAEmB,QAAgB;;YAS1E,2BAAA,mBAAA,gBAAA,WAAA,OAAME;;;;oBARX;;wBAAMH,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA,wFAAwF;oBACxF,4FAA4F;oBAC5F;;wBAAMW,mBAAmBC,MAAMC;;;oBAA/B;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMY,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBgB,IAAAA,wBAAc,EAACtB;;;2BAAhC,6BAAA,QAAA;;;;oBAAMqB,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACL;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,kBAAkBxB;;;oBAAtC;;;oBACjD;;wBAAM8B,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,YAAYQ,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASC,cAAc7B,EAAgB;IACrC,IAAM8B,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFhC,GAAGO,IAAI,CAAC;YACR;QACF,EAAE,OAAOC,KAAK;YACZ,IAAIuB,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BG,IAAI,CAAC,AAACzB,IAAcE,OAAO,GAAG,MAAMF;YAC9F0B,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,SAAeC,QAAQC,MAAc,EAAEtC,GAAmB;;YAClDD,IASEmB,MAGAC;;YAZFpB,KAAK,IAAIwC,wBAAY,CAACD;YAC5B,IAAI;gBACF,6FAA6F;gBAC7F,yFAAyF;gBACzFvC,GAAGO,IAAI,CAAC;gBACR,yFAAyF;gBACzF,wEAAwE;gBACxEsB,cAAc7B;gBACdyC,IAAAA,iCAAiB,EAACzC,IAAIG,IAAAA,wBAAe,EAACF,SAASG;gBACzCe,OAAOuB,IAAAA,8BAAgB,EAAC1C;gBAC9B,2FAA2F;gBAC3F,8FAA8F;gBACxFoB,WAAWrB,gBAAgBC,IAAIC;gBACrC;;oBAAO;wBAAE0C,QAAQ;4BAAE3C,IAAAA;4BAAIoB,UAAAA;wBAAS;wBAAGD,MAAAA;oBAAK;;YAC1C,EAAE,OAAOX,KAAK;gBACZ,4FAA4F;gBAC5F,0CAA0C;gBAC1CR,GAAG4C,KAAK;gBACR,MAAMpC;YACR;;;;;IACF;;AAEA,SAAeoC,MAAMD,MAAoB;;;YACvCA,OAAO3C,EAAE,CAAC4C,KAAK;;;;;IACjB;;AAEA,6FAA6F;AAC7F,gGAAgG;AAChG,SAAeC,eAAeF,MAAoB,EAAExB,IAAgB,EAAE2B,IAAY,EAAEC,WAAwB;;;;;oBAC1G,IAAI,CAAEA,CAAAA,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,WAAU,GAAI;;wBAAO;;oBACrEC,QAAQC,KAAK,CAAC;oBACd,2FAA2F;oBAC3F,yFAAyF;oBACzF;;wBAAMC,IAAAA,8BAAe,EACnBjC,MACA;;;;;4CACE;;gDAAMA,KAAKZ,IAAI,CAAC;;;4CAAhB;4CACA;;gDAAMW,mBAAmBC,MAAMwB,OAAOvB,QAAQ;;;4CAA9C;;;;;;4BACF;2BACAiC,0BAAW;;;oBANb;oBAQA;;wBAAO;;;;IACT;;AAEA,oGAAoG;AACpG,qIAAqI;AACrI,SAAeC,kBAAkBC,OAAqB,EAAEpC,IAAgB,EAAElB,GAAW,EAAEuD,OAAe,EAAEC,YAAoB;;YACpHC,MACAC,UACAC,OACAC,YACAC,UACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAETF,WADwCG,YAAhCC,KAAeC;;;;oBAPZ;;wBAAMhD,KAAKN,OAAO,CAAC;;;oBAA1B6C,OAAO;wBACKU;oBAAM;;wBAAMV,KAAKW,GAAG;;;oBAAhCV,QAAQ,IAAA,CAAA,EAAA,MAAIS;;wBAAM,cAA8CE,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFZ,QAAQa,IAAAA,mBAAS,EAACxE,KAAKuD,SAASkB,MAAM,CAAC,SAACC;+BAAMhB,MAAMV,GAAG,CAAC0B,EAAEC,OAAO;;oBACjEf,aAAa1D,IAAAA,wBAAe,EAACF,SAASG;oBACtC0D;oBACAC;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAcH,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAfI,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1CE,MAAgCD,WAAhCC,KAAeC,eAAiBF,WAA3BH;4BACbA,CAAAA,YAAAA,UAASe,IAAI,OAAbf,WAAc,qBAAGK;4BACjBJ,KAAKc,IAAI,CAACC,IAAAA,uBAAU,EAACZ,KAAKL;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMT,IAAAA,8BAAe,EACnBjC,MACA;;;;;iDACM4C,CAAAA,KAAKgB,MAAM,GAAG,CAAA,GAAdhB;;;;4CAAiB;;gDAAM5C,KAAK6D,QAAQ,CAACC,+BAAkB,EAAElB;;;4CAAxC;;;;;;;;4BACvB;2BACAV,0BAAW;;;oBALb;oBAOA;;wBAAM3B,IAAAA,iBAAO,EAACP,MAAM,YAAYsC;;;oBAAhC;oBACA;;wBAAOK;;;;IACT;;AAEA,6FAA6F;AAC7F,8DAA8D;AAC9D,SAASoB,mBAAmBvC,MAAoB;IAC9C,IAAMwC,SAASxE,eAAegC,OAAO3C,EAAE;IACvC,IAAImF,WAAW,QAAQA,WAAWxC,OAAOvB,QAAQ,EAAE,OAAO;IAC1D,OAAO;AACT;AAEA,SAAegE,aAAazC,MAAoB,EAAExB,IAAgB,EAAElB,GAAW;;;;;oBAC7E,+FAA+F;oBAC/F,+EAA+E;oBAC/E;;wBAAMmD,IAAAA,8BAAe,EAACjC,MAAM;mCAAME,mBAAmBF,MAAMlB,KAAK0C,OAAOvB,QAAQ;2BAAGiC,0BAAW;;;oBAA7F;;;;;;IACF;;AAEA,SAAegC,sBAAsB1C,MAAoB,EAAE2C,KAAiB,EAAEC,EAAU;;;YACtF5C,OAAO3C,EAAE,CAACO,IAAI,CAAC,AAAC,yBAA2B,OAAHgF;;;;;IAC1C;;AAGO,IAAM1F,oBAA+C;IAC1D2F,UAAU9F;IACV+F,eAAe9F;IACf+F,kBAAkBC,0BAAa;IAC/BrD,SAAAA;IACAM,OAAAA;IACAC,gBAAAA;IACAS,mBAAAA;IACA4B,oBAAAA;IACAE,cAAAA;IACAC,uBAAAA;IACAO,aAAa,SAAbA,YAAcjD,QAAQxB,MAAMlB;eAAQ2F,IAAAA,oBAAW,EAACjD,OAAO3C,EAAE,EAAEmB,MAAMlB;;AACnE;AAEO,SAAeL,WAAWK,GAAmB;;;YAClD;;gBAAO4F,IAAAA,uBAAe,EAAC5F,KAAKJ;;;IAC9B"}

package/dist/cjs/store/sqlite/store.d.cts

@@ -2,4 +2,4 @@ import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;

package/dist/cjs/store/sqlite/store.d.ts

@@ -2,4 +2,4 @@ import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;

package/dist/cjs/store/sqlite/store.js

@@ -17,13 +17,11 @@ _export(exports, {
     }
 });
 var _indexts = require("../../config/index.js");
-var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets1 = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -371,7 +369,7 @@ var CAPABILITIES = new Set([
     'vectors',
     'sql-functions'
 ]);
-function createStore(db, conn, cfg, baseDir) {
+function createStore(db, conn, cfg) {
     return {
         name: 'sqlite',
         capabilities: CAPABILITIES,
@@ -431,16 +429,6 @@ function createStore(db, conn, cfg, baseDir) {
                 });
             })();
         },
-        reconcile: function reconcile() {
-            return _async_to_generator(function() {
-                return _ts_generator(this, function(_state) {
-                    return [
-                        2,
-                        (0, _reconcilets.reconcile)(conn, cfg, baseDir, _reconcilets1.sqliteDialect)
-                    ];
-                });
-            })();
-        },
         docs: {
             columns: function columns() {
                 return _async_to_generator(function() {