sensemaking 0.17.0 → 0.18.0

package/dist/cjs/store/sqlite/open.js

@@ -0,0 +1,604 @@
+// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
+// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get DB_FILENAME () {
+        return DB_FILENAME;
+    },
+    get SCHEMA_VERSION () {
+        return SCHEMA_VERSION;
+    },
+    get clearCache () {
+        return clearCache;
+    },
+    get docCount () {
+        return docCount;
+    },
+    get openSqlite () {
+        return openSqlite;
+    }
+});
+var _nodefs = require("node:fs");
+var _nodepath = require("node:path");
+var _nodesqlite = require("node:sqlite");
+var _indexts = require("../../config/index.js");
+var _errorsts = require("../../errors.js");
+var _indexts1 = require("../../features/index.js");
+var _sharedts = require("../shared.js");
+var _connectionts = require("./connection.js");
+var _reconcilets = require("./reconcile.js");
+var _sqlfunctionsts = require("./sql-functions.js");
+var _storets = require("./store.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var DB_FILENAME = 'cache.db';
+var SCHEMA_VERSION = '18';
+// Stemming is English-only, but the segmentation underneath it is what decides coverage:
+// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
+// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
+// such a tree picks trigram instead.
+var DEFAULT_TOKENIZE = 'porter unicode61';
+// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a
+// configured value has to be concatenated. Probing a throwaway table is what makes that safe
+// and is also the whole validation: anything the linked SQLite accepts passes, anything else
+// fails here with SQLite's own message rather than against the real table. It means no table
+// of which version added which tokenizer has to be maintained.
+function resolveTokenize(db, cfg) {
+    var configured = (0, _indexts.contentTokenize)(cfg);
+    if (configured === undefined) return DEFAULT_TOKENIZE;
+    var literal = configured.replace(/'/g, "''");
+    try {
+        db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');
+        db.exec("CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '".concat(literal, "')"));
+        db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');
+    } catch (err) {
+        throw new _errorsts.SenseError('CONFIG_INVALID', 'content.tokenize "'.concat(configured, '" is not a tokenizer this SQLite accepts (').concat(err.message, "); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options"));
+    }
+    return literal;
+}
+// The tokenizer the content table was actually built with, from its own DDL -- the one
+// record that cannot desynchronize from the table. NULL when the table does not exist yet.
+function storedTokenize(db) {
+    var row = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'content'").get();
+    if (!row) return null;
+    var m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
+    return m ? m[1] : null;
+}
+// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
+// can't dump file text into context. Features add their own tables after the core ones.
+function ensureSchema(conn, cfg, tokenize) {
+    return _async_to_generator(function() {
+        var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)')
+                    ];
+                case 1:
+                    _state.sent();
+                    // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL
+                    // against the resolved tokenizer before this runs, so a stale table is already gone by now.
+                    // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
+                    // snippet(content, 2, ...) are documented against the first three columns and keep working
+                    // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
+                    // runs for text that needs it, and an empty string for text that does not.
+                    return [
+                        4,
+                        conn.exec("CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '".concat(tokenize, "')"))
+                    ];
+                case 2:
+                    _state.sent();
+                    // Coverage, not ownership: a path can appear under several presets. path leads the PK so the
+                    // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
+                    ];
+                case 3:
+                    _state.sent();
+                    return [
+                        4,
+                        conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
+                    ];
+                case 4:
+                    _state.sent();
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 5;
+                case 5:
+                    _state.trys.push([
+                        5,
+                        10,
+                        11,
+                        12
+                    ]);
+                    _iterator = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
+                    _state.label = 6;
+                case 6:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        9
+                    ];
+                    feature = _step.value;
+                    return [
+                        4,
+                        feature.schema(conn)
+                    ];
+                case 7:
+                    _state.sent();
+                    _state.label = 8;
+                case 8:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        6
+                    ];
+                case 9:
+                    return [
+                        3,
+                        12
+                    ];
+                case 10:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        12
+                    ];
+                case 11:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 12:
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'schema_version')
+                    ];
+                case 13:
+                    if (!(_state.sent() === null)) return [
+                        3,
+                        15
+                    ];
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
+                    ];
+                case 14:
+                    _state.sent();
+                    _state.label = 15;
+                case 15:
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'features')
+                    ];
+                case 16:
+                    if (!(_state.sent() === null)) return [
+                        3,
+                        18
+                    ];
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
+                    ];
+                case 17:
+                    _state.sent();
+                    _state.label = 18;
+                case 18:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function connect(cfg) {
+    return _async_to_generator(function() {
+        var stateDir, dbPath, db, closed, _ref, conn, tokenize, version, features, wantFeatures, tokenizeOnlyRebuild, changedKeys, changed, stored, rebuildWarnings, recordedMaxMs, _ref1, parsed, warnings, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
+                    (0, _nodefs.mkdirSync)(stateDir, {
+                        recursive: true
+                    });
+                    dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
+                    db = new _nodesqlite.DatabaseSync(dbPath);
+                    // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,
+                    // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.
+                    // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.
+                    closed = false;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        14,
+                        ,
+                        15
+                    ]);
+                    db.exec('PRAGMA journal_mode = WAL');
+                    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
+                    // that outwaits it still fails loudly.
+                    db.exec('PRAGMA busy_timeout = 30000');
+                    (0, _sqlfunctionsts.registerFunctions)(db, (0, _indexts.contentTokenize)(cfg) === undefined);
+                    conn = (0, _connectionts.createConnection)(db);
+                    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
+                    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd
+                    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.
+                    tokenize = resolveTokenize(db, cfg);
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'schema_version')
+                    ];
+                case 2:
+                    version = _state.sent();
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'features')
+                    ];
+                case 3:
+                    features = _state.sent();
+                    wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
+                    tokenizeOnlyRebuild = false;
+                    if (!(version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures)) return [
+                        3,
+                        7
+                    ];
+                    // Indexing derives from presets, so a config edit rebuilding the cache must say so and
+                    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
+                    if (version !== null && version !== SCHEMA_VERSION) {
+                        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
+                        closed = true;
+                        db.close();
+                        clearCache(cfg);
+                        return [
+                            2,
+                            connect(cfg)
+                        ];
+                    }
+                    changedKeys = (0, _reconcilets.changedSignatureKeys)(features !== null && features !== void 0 ? features : '', wantFeatures);
+                    if (!(changedKeys.size === 1 && changedKeys.has('tokenize'))) return [
+                        3,
+                        4
+                    ];
+                    console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
+                    db.exec('DROP TABLE content');
+                    tokenizeOnlyRebuild = true;
+                    return [
+                        3,
+                        7
+                    ];
+                case 4:
+                    if (!(changedKeys.size === 1 && changedKeys.has('embed') && (0, _reconcilets.embedIdentityAdopted)(features !== null && features !== void 0 ? features : '', wantFeatures))) return [
+                        3,
+                        6
+                    ];
+                    // First sight of a resolved weight identity: the model itself hasn't changed, so
+                    // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.
+                    console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
+                    ];
+                case 5:
+                    _state.sent();
+                    return [
+                        3,
+                        7
+                    ];
+                case 6:
+                    changed = (0, _reconcilets.signatureDiff)(features !== null && features !== void 0 ? features : '', wantFeatures);
+                    console.error("sense: config change (".concat(changed, ") rebuilds the index"));
+                    closed = true;
+                    db.close();
+                    clearCache(cfg);
+                    return [
+                        2,
+                        connect(cfg)
+                    ];
+                case 7:
+                    // Meta can lie after a crash between table creation and the signature write; the table's
+                    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
+                    // just dropped content above, so storedTokenize sees no table and this guard is skipped
+                    // naturally rather than needing its own case.
+                    stored = storedTokenize(db);
+                    if (stored !== null && stored !== tokenize) {
+                        console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
+                        closed = true;
+                        db.close();
+                        clearCache(cfg);
+                        return [
+                            2,
+                            connect(cfg)
+                        ];
+                    }
+                    return [
+                        4,
+                        ensureSchema(conn, cfg, tokenize)
+                    ];
+                case 8:
+                    _state.sent();
+                    rebuildWarnings = [];
+                    if (!tokenizeOnlyRebuild) return [
+                        3,
+                        11
+                    ];
+                    return [
+                        4,
+                        (0, _reconcilets.rebuildContentTable)(conn, cfg, cfg.baseDir)
+                    ];
+                case 9:
+                    rebuildWarnings = _state.sent();
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
+                    ];
+                case 10:
+                    _state.sent();
+                    _state.label = 11;
+                case 11:
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
+                    ];
+                case 12:
+                    recordedMaxMs = Number.apply(void 0, [
+                        (_ref = _state.sent()) !== null && _ref !== void 0 ? _ref : '0'
+                    ]);
+                    db.exec("PRAGMA busy_timeout = ".concat(Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)));
+                    return [
+                        4,
+                        (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir)
+                    ];
+                case 13:
+                    _ref1 = _state.sent(), parsed = _ref1.parsed, warnings = _ref1.warnings;
+                    return [
+                        2,
+                        {
+                            db: db,
+                            conn: conn,
+                            cfg: cfg,
+                            dbPath: dbPath,
+                            parsed: parsed,
+                            warnings: _to_consumable_array(rebuildWarnings).concat(_to_consumable_array(warnings))
+                        }
+                    ];
+                case 14:
+                    err = _state.sent();
+                    if (!closed) db.close();
+                    throw err;
+                case 15:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function openSqlite(cfg) {
+    return _async_to_generator(function() {
+        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        connect(cfg)
+                    ];
+                case 1:
+                    _ref = _state.sent(), db = _ref.db, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
+                    return [
+                        2,
+                        {
+                            store: (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir),
+                            cfg: resolvedCfg,
+                            dbPath: dbPath,
+                            parsed: parsed,
+                            warnings: warnings
+                        }
+                    ];
+            }
+        });
+    })();
+}
+function docCount(store) {
+    return _async_to_generator(function() {
+        var stmt;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        store.prepare('SELECT COUNT(*) AS n FROM frontmatter')
+                    ];
+                case 1:
+                    stmt = _state.sent();
+                    return [
+                        4,
+                        stmt.get()
+                    ];
+                case 2:
+                    return [
+                        2,
+                        _state.sent().n
+                    ];
+            }
+        });
+    })();
+}
+function clearCache(cfg) {
+    (0, _nodefs.rmSync)((0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR), {
+        recursive: true,
+        force: true
+    });
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/sqlite/open.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n  // snippet(content, 2, ...) are documented against the first three columns and keep working\n  // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n  // runs for text that needs it, and an empty string for text that does not.\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,\n  // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.\n  // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n      // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n      // edit, an embed model change -- still takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        db.exec('DROP TABLE content');\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's\n    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n    // just dropped content above, so storedTokenize sees no table and this guard is skipped\n    // naturally rather than needing its own case.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","openSqlite","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","contentTokenize","undefined","literal","replace","exec","err","SenseError","message","storedTokenize","row","prepare","get","m","sql","match","ensureSchema","conn","tokenize","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","changedKeys","changed","stored","rebuildWarnings","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","DatabaseSync","registerFunctions","createConnection","console","error","close","changedSignatureKeys","size","has","embedIdentityAdopted","signatureDiff","rebuildContentTable","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","stmt","n","rmSync","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAepFA;eAAAA;;QAGAC;eAAAA;;QA2LGC;eAAAA;;QATMC;eAAAA;;QALAC;eAAAA;;;sBA9LY;wBACb;0BACQ;uBAEgC;wBAClC;wBACc;wBACR;4BAEA;2BACyE;8BACxE;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMJ,cAAc;AAGpB,IAAMC,iBAAiB;AAmB9B,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,IAAMI,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,IAAMC,aAAaC,IAAAA,wBAAe,EAACF;IACnC,IAAIC,eAAeE,WAAW,OAAON;IACrC,IAAMO,UAAUH,WAAWI,OAAO,CAAC,MAAM;IACzC,IAAI;QACFN,GAAGO,IAAI,CAAC;QACRP,GAAGO,IAAI,CAAC,AAAC,4EAAmF,OAARF,SAAQ;QAC5FL,GAAGO,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIC,oBAAU,CAAC,kBAAkB,AAAC,qBAA2E,OAAvDP,YAAW,8CAAmE,OAAvB,AAACM,IAAcE,OAAO,EAAC;IAC5I;IACA,OAAOL;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASM,eAAeX,EAAgB;IACtC,IAAMY,MAAMZ,GAAGa,OAAO,CAAC,wDAAwDC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,IAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeG,aAAaC,IAAgB,EAAElB,GAAW,EAAEmB,QAAgB;;YAapE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAZX;;wBAAMF,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA,wFAAwF;oBACxF,4FAA4F;oBAC5F,4FAA4F;oBAC5F,2FAA2F;oBAC3F,2FAA2F;oBAC3F,2EAA2E;oBAC3E;;wBAAMY,KAAKZ,IAAI,CAAC,AAAC,6IAAqJ,OAATa,UAAS;;;oBAAtK;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMY,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBe,IAAAA,wBAAc,EAACrB;;;2BAAhC,6BAAA,QAAA;;;;oBAAMoB,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACJ;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMK,IAAAA,iBAAO,EAACL,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM,kBAAkBzB;;;oBAAtC;;;oBACjD;;wBAAM8B,IAAAA,iBAAO,EAACL,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM,YAAYO,IAAAA,yBAAgB,EAACzB,KAAK0B,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQ3B,GAAmB;;YAClC4B,UAEAC,QAEA9B,IAIF+B,QA6E4B,MAtExBZ,MAMAC,UAIAY,SACAC,UACAC,cACFC,qBAWIC,aAcEC,SAaJC,QAWFC,iBAQEC,eAGuB,OAArBC,QAAQC,UAGTlC;;;;oBA3FHqB,WAAWc,IAAAA,cAAI,EAAC1C,IAAI2C,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACjB,UAAU;wBAAEkB,WAAW;oBAAK;oBAChCjB,SAASa,IAAAA,cAAI,EAACd,UAAUpC;oBAExBO,KAAK,IAAIgD,wBAAY,CAAClB;oBAC5B,8FAA8F;oBAC9F,6FAA6F;oBAC7F,8FAA8F;oBAC1FC,SAAS;;;;;;;;;oBAEX/B,GAAGO,IAAI,CAAC;oBACR,yFAAyF;oBACzF,uCAAuC;oBACvCP,GAAGO,IAAI,CAAC;oBACR0C,IAAAA,iCAAiB,EAACjD,IAAIG,IAAAA,wBAAe,EAACF,SAASG;oBACzCe,OAAO+B,IAAAA,8BAAgB,EAAClD;oBAE9BA,GAAGO,IAAI,CAAC;oBAER,2FAA2F;oBAC3F,8FAA8F;oBACxFa,WAAWrB,gBAAgBC,IAAIC;oBAIrB;;wBAAMuB,IAAAA,iBAAO,EAACL,MAAM;;;oBAA9Ba,UAAU;oBACC;;wBAAMR,IAAAA,iBAAO,EAACL,MAAM;;;oBAA/Bc,WAAW;oBACXC,eAAeR,IAAAA,yBAAgB,EAACzB,KAAK0B,kBAAQ;oBAC/CQ,sBAAsB;yBACtB,CAAA,AAACH,YAAY,QAAQA,YAAYtC,kBAAoBuC,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,uFAAuF;oBACvF,wFAAwF;oBACxF,IAAIF,YAAY,QAAQA,YAAYtC,gBAAgB;wBAClDyD,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACT/B,GAAGqD,KAAK;wBACR1D,WAAWM;wBACX;;4BAAO2B,QAAQ3B;;oBACjB;oBACMmC,cAAckB,IAAAA,iCAAoB,EAACrB,qBAAAA,sBAAAA,WAAY,IAAIC;yBAIrDE,CAAAA,YAAYmB,IAAI,KAAK,KAAKnB,YAAYoB,GAAG,CAAC,WAAU,GAApDpB;;;;oBACFe,QAAQC,KAAK,CAAC;oBACdpD,GAAGO,IAAI,CAAC;oBACR4B,sBAAsB;;;;;;yBACbC,CAAAA,YAAYmB,IAAI,KAAK,KAAKnB,YAAYoB,GAAG,CAAC,YAAYC,IAAAA,iCAAoB,EAACxB,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGE;;;;oBACT,iFAAiF;oBACjF,wFAAwF;oBACxFe,QAAQC,KAAK,CAAC;oBACd;;wBAAM3B,IAAAA,iBAAO,EAACN,MAAM,YAAYe;;;oBAAhC;;;;;;oBAEMG,UAAUqB,IAAAA,0BAAa,EAACzB,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9CiB,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARf,SAAQ;oBAC/CN,SAAS;oBACT/B,GAAGqD,KAAK;oBACR1D,WAAWM;oBACX;;wBAAO2B,QAAQ3B;;;oBAInB,yFAAyF;oBACzF,6FAA6F;oBAC7F,wFAAwF;oBACxF,8CAA8C;oBACxCqC,SAAS3B,eAAeX;oBAC9B,IAAIsC,WAAW,QAAQA,WAAWlB,UAAU;wBAC1C+B,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACT/B,GAAGqD,KAAK;wBACR1D,WAAWM;wBACX;;4BAAO2B,QAAQ3B;;oBACjB;oBAEA;;wBAAMiB,aAAaC,MAAMlB,KAAKmB;;;oBAA9B;oBAEImB;yBACAJ,qBAAAA;;;;oBACgB;;wBAAMwB,IAAAA,gCAAmB,EAACxC,MAAMlB,KAAKA,IAAI2C,OAAO;;;oBAAlEL,kBAAkB;oBAClB;;wBAAMd,IAAAA,iBAAO,EAACN,MAAM,YAAYe;;;oBAAhC;;;oBAK4B;;wBAAMV,IAAAA,iBAAO,EAACL,MAAM;;;oBAA5CqB,gBAAgBoB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E5D,GAAGO,IAAI,CAAC,AAAC,yBAA8E,OAAtDsD,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIvB,gBAAgB;oBAEjD;;wBAAMwB,IAAAA,sBAAS,EAAC7C,MAAMlB,KAAKA,IAAI2C,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAE1C,IAAAA;4BAAImB,MAAAA;4BAAMlB,KAAAA;4BAAK6B,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAU,AAAC,qBAAGH,wBAAiB,qBAAGG;wBAAU;;;oBAC7ElC;oBACP,IAAI,CAACuB,QAAQ/B,GAAGqD,KAAK;oBACrB,MAAM7C;;;;;;;IAEV;;AAIO,SAAeX,WAAWI,GAAmB;;YACe,MAAzDD,IAAImB,MAAW8C,aAAanC,QAAQW,QAAQC;;;;oBAAa;;wBAAMd,QAAQ3B;;;oBAAd,OAAA,eAAzDD,KAAyD,KAAzDA,IAAImB,OAAqD,KAArDA,MAAW8C,cAA0C,KAA/ChE,KAAkB6B,SAA6B,KAA7BA,QAAQW,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBACpD;;wBAAO;4BAAEwB,OAAOC,IAAAA,oBAAW,EAACnE,IAAImB,MAAM8C,aAAaA,YAAYrB,OAAO;4BAAG3C,KAAKgE;4BAAanC,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACtH;;AAEO,SAAe9C,SAASsE,KAAY;;YACnCE;;;;oBAAO;;wBAAMF,MAAMrD,OAAO,CAAC;;;oBAA3BuD,OAAO;oBACJ;;wBAAMA,KAAKtD,GAAG;;;oBAAvB;;wBAAS,cAAoCuD,CAAC;;;;IAChD;;AAMO,SAAS1E,WAAWM,GAAmB;IAC5CqE,IAAAA,cAAM,EAAC3B,IAAAA,cAAI,EAAC1C,IAAI2C,OAAO,EAAEC,kBAAS,GAAG;QAAEE,WAAW;QAAMwB,OAAO;IAAK;AACtE"}

package/dist/cjs/store/sqlite/reconcile.d.cts

@@ -0,0 +1,10 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
+    parsed: number;
+    warnings: string[];
+}>;
+export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function signatureDiff(before: string, after: string): string;
+export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;

package/dist/cjs/store/sqlite/reconcile.d.ts

@@ -0,0 +1,10 @@
+import type { Config } from '../../config/index.js';
+import type { Connection } from '../types.js';
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
+    parsed: number;
+    warnings: string[];
+}>;
+export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
+export declare function signatureDiff(before: string, after: string): string;
+export declare function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]>;