sensemaking 0.19.2 → 0.21.0

package/dist/cjs/store/open.js

@@ -0,0 +1,608 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get docCount () {
+        return docCount;
+    },
+    get openWithBuilder () {
+        return openWithBuilder;
+    },
+    get openWithDialect () {
+        return openWithDialect;
+    }
+});
+var _nodefs = require("node:fs");
+var _nodepath = require("node:path");
+var _indexts = require("../config/index.js");
+var _handoffts = require("../embed/handoff.js");
+var _errorsts = require("../errors.js");
+var _indexts1 = require("../features/index.js");
+var _builderts = require("./builder.js");
+var _cachets = require("./cache.js");
+var _embedscopets = require("./embed-scope.js");
+var _featurescopets = require("./feature-scope.js");
+var _presetscopets = require("./preset-scope.js");
+var _sharedts = require("./shared.js");
+var _signaturets = require("./signature.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+// duckdb and turso hold the cache file for their connection's whole life, not for a transaction,
+// so a second command waits on the first command finishing. Bounded rather than open-ended: this
+// clears a warm query's collision (~100ms) and still fails loudly behind a cold build instead of
+// looking hung.
+var LOCK_RETRY_MS = 5000;
+var LOCK_POLL_MS = 50;
+function connectUnlocked(dbPath, cfg, dialect) {
+    return _async_to_generator(function() {
+        var deadline, err, _dialect_isLocked;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    deadline = Date.now() + LOCK_RETRY_MS;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        3,
+                        ,
+                        5
+                    ]);
+                    return [
+                        4,
+                        dialect.connect(dbPath, cfg)
+                    ];
+                case 2:
+                    return [
+                        2,
+                        _state.sent()
+                    ];
+                case 3:
+                    err = _state.sent();
+                    if (!((_dialect_isLocked = dialect.isLocked) === null || _dialect_isLocked === void 0 ? void 0 : _dialect_isLocked.call(dialect, err))) throw err;
+                    if (Date.now() >= deadline) {
+                        throw new _errorsts.SenseError('STORE_BUSY', "another sense process is using this tree's ".concat(cfg.store, " cache (").concat(dbPath, ") and did not release it within ").concat(LOCK_RETRY_MS / 1000, 's; wait for that command to finish, or set "store" to "sqlite" in sense.config.json, which serves concurrent commands'));
+                    }
+                    return [
+                        4,
+                        new Promise(function(resolve) {
+                            return setTimeout(resolve, LOCK_POLL_MS);
+                        })
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        3,
+                        5
+                    ];
+                case 5:
+                    return [
+                        3,
+                        1
+                    ];
+                case 6:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function connectWithDialect(cfg, dialect, keepBuilderOpen) {
+    return _async_to_generator(function() {
+        var stateDir, dbPath, _ref, handle, conn, closed, builder, version, features, wantFeatures, forcedPaths, embedInvalidate, featureToggles, changedKeys, presetForced, embedKind, toggles, changed, changed1, changed2, _ref1, recordedMaxMs, embedParsed, _tmp, featureParsed, _tmp1, _ref2, parsed, warnings, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
+                    (0, _nodefs.mkdirSync)(stateDir, {
+                        recursive: true
+                    });
+                    dbPath = (0, _nodepath.join)(stateDir, dialect.filename);
+                    return [
+                        4,
+                        connectUnlocked(dbPath, cfg, dialect)
+                    ];
+                case 1:
+                    _ref = _state.sent(), handle = _ref.handle, conn = _ref.conn;
+                    // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on
+                    // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.
+                    closed = false;
+                    // Created here, not inside reconcile(): its pool must survive across every build() call this
+                    // connection sees (a watcher's repeated ticks), not just the one below.
+                    builder = (0, _builderts.createBuilder)(conn, cfg, cfg.baseDir, dialect.reconcileDialect);
+                    _state.label = 2;
+                case 2:
+                    _state.trys.push([
+                        2,
+                        31,
+                        ,
+                        35
+                    ]);
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)')
+                    ];
+                case 3:
+                    _state.sent();
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'schema_version')
+                    ];
+                case 4:
+                    version = _state.sent();
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'features')
+                    ];
+                case 5:
+                    features = _state.sent();
+                    wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
+                    if (!(version !== null && version !== dialect.schemaVersion || features !== null && features !== wantFeatures)) return [
+                        3,
+                        17
+                    ];
+                    if (!(version !== null && version !== dialect.schemaVersion)) return [
+                        3,
+                        7
+                    ];
+                    console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
+                    // Close before clearCache deletes the files out from under the still-open handle.
+                    closed = true;
+                    return [
+                        4,
+                        dialect.close(handle)
+                    ];
+                case 6:
+                    _state.sent();
+                    (0, _cachets.clearCache)(cfg);
+                    return [
+                        2,
+                        connectWithDialect(cfg, dialect, keepBuilderOpen)
+                    ];
+                case 7:
+                    changedKeys = (0, _signaturets.changedSignatureKeys)(features !== null && features !== void 0 ? features : '', wantFeatures);
+                    // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed
+                    // preset's stored segment can't be parsed back into old include/exclude/vectors -- either
+                    // way the branch below is skipped and the full rebuild stays the fallback.
+                    presetForced = (0, _presetscopets.isPresetOnlyChange)(changedKeys) ? (0, _presetscopets.forcedPresetPaths)(cfg, cfg.baseDir, features !== null && features !== void 0 ? features : '', changedKeys) : null;
+                    // Null unless exactly one segment changed and it decomposes into a recognised embed-only
+                    // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.
+                    embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? (0, _embedscopets.classifyEmbedChange)(features !== null && features !== void 0 ? features : '', wantFeatures) : null;
+                    // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed
+                    // feature's stored segment can't be parsed back into old/new on-off -- either way the branch
+                    // below is skipped and the full rebuild stays the fallback.
+                    toggles = (0, _featurescopets.isFeatureOnlyChange)(changedKeys) ? (0, _featurescopets.classifyFeatureToggles)(features !== null && features !== void 0 ? features : '', wantFeatures, changedKeys) : null;
+                    if (!(changedKeys.size === 1 && changedKeys.has('embed') && (0, _signaturets.embedIdentityAdopted)(features !== null && features !== void 0 ? features : '', wantFeatures))) return [
+                        3,
+                        9
+                    ];
+                    // First sight of a resolved weight identity: the model itself hasn't changed, so
+                    // adopt it into meta with no rebuild and no re-embed.
+                    console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
+                    ];
+                case 8:
+                    _state.sent();
+                    return [
+                        3,
+                        17
+                    ];
+                case 9:
+                    if (!(embedKind !== null)) return [
+                        3,
+                        11
+                    ];
+                    // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither
+                    // case needs any file reparsed for any feature but embed itself.
+                    embedInvalidate = embedKind;
+                    console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
+                    ];
+                case 10:
+                    _state.sent();
+                    return [
+                        3,
+                        17
+                    ];
+                case 11:
+                    if (!(presetForced !== null)) return [
+                        3,
+                        13
+                    ];
+                    // Membership only: which files a preset covers moved, not a file's own content, a
+                    // global feature, or the embed model. reconcile() already knows how to
+                    // add, update, and remove a file correctly (every cross-feature cascade included); it
+                    // just needs telling which unchanged files to treat as touched (forcedPaths, above).
+                    forcedPaths = presetForced;
+                    changed = (0, _signaturets.signatureDiff)(features !== null && features !== void 0 ? features : '', wantFeatures);
+                    console.error("sense: config change (".concat(changed, ") reparses only the files it affects"));
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
+                    ];
+                case 12:
+                    _state.sent();
+                    return [
+                        3,
+                        17
+                    ];
+                case 13:
+                    if (!(toggles !== null)) return [
+                        3,
+                        15
+                    ];
+                    // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's
+                    // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a
+                    // feature turning back on, fully re-derives across every indexed file rather than
+                    // trusting rows written before or during that window.
+                    featureToggles = toggles;
+                    changed1 = (0, _signaturets.signatureDiff)(features !== null && features !== void 0 ? features : '', wantFeatures);
+                    console.error("sense: config change (".concat(changed1, ") reparses only the feature it affects"));
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
+                    ];
+                case 14:
+                    _state.sent();
+                    return [
+                        3,
+                        17
+                    ];
+                case 15:
+                    changed2 = (0, _signaturets.signatureDiff)(features !== null && features !== void 0 ? features : '', wantFeatures);
+                    console.error("sense: config change (".concat(changed2, ") rebuilds the index"));
+                    closed = true;
+                    return [
+                        4,
+                        dialect.close(handle)
+                    ];
+                case 16:
+                    _state.sent();
+                    (0, _cachets.clearCache)(cfg);
+                    return [
+                        2,
+                        connectWithDialect(cfg, dialect, keepBuilderOpen)
+                    ];
+                case 17:
+                    return [
+                        4,
+                        dialect.ensureSchema(handle, conn, cfg)
+                    ];
+                case 18:
+                    _state.sent();
+                    if (!dialect.setDerivedBusyTimeout) return [
+                        3,
+                        21
+                    ];
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
+                    ];
+                case 19:
+                    recordedMaxMs = Number.apply(void 0, [
+                        (_ref1 = _state.sent()) !== null && _ref1 !== void 0 ? _ref1 : '0'
+                    ]);
+                    return [
+                        4,
+                        dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600000))
+                    ];
+                case 20:
+                    _state.sent();
+                    _state.label = 21;
+                case 21:
+                    if (!embedInvalidate) return [
+                        3,
+                        23
+                    ];
+                    return [
+                        4,
+                        builder.invalidate(embedInvalidate)
+                    ];
+                case 22:
+                    _tmp = _state.sent().parsed;
+                    return [
+                        3,
+                        24
+                    ];
+                case 23:
+                    _tmp = 0;
+                    _state.label = 24;
+                case 24:
+                    embedParsed = _tmp;
+                    if (!featureToggles) return [
+                        3,
+                        26
+                    ];
+                    return [
+                        4,
+                        builder.invalidateFeatures(featureToggles)
+                    ];
+                case 25:
+                    _tmp1 = _state.sent().parsed;
+                    return [
+                        3,
+                        27
+                    ];
+                case 26:
+                    _tmp1 = 0;
+                    _state.label = 27;
+                case 27:
+                    featureParsed = _tmp1;
+                    return [
+                        4,
+                        builder.build(forcedPaths)
+                    ];
+                case 28:
+                    _ref2 = _state.sent(), parsed = _ref2.parsed, warnings = _ref2.warnings;
+                    if (!!keepBuilderOpen) return [
+                        3,
+                        30
+                    ];
+                    return [
+                        4,
+                        builder.close()
+                    ];
+                case 29:
+                    _state.sent();
+                    _state.label = 30;
+                case 30:
+                    return [
+                        2,
+                        {
+                            handle: handle,
+                            conn: conn,
+                            cfg: cfg,
+                            dbPath: dbPath,
+                            parsed: parsed + embedParsed + featureParsed,
+                            warnings: warnings,
+                            builder: builder
+                        }
+                    ];
+                case 31:
+                    err = _state.sent();
+                    return [
+                        4,
+                        builder.close()
+                    ];
+                case 32:
+                    _state.sent();
+                    if (!!closed) return [
+                        3,
+                        34
+                    ];
+                    return [
+                        4,
+                        dialect.close(handle)
+                    ];
+                case 33:
+                    _state.sent();
+                    _state.label = 34;
+                case 34:
+                    throw err;
+                case 35:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function openWithDialect(cfg, dialect) {
+    return _async_to_generator(function() {
+        var _ref, handle, conn, resolvedCfg, dbPath, parsed, warnings, store;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        connectWithDialect(cfg, dialect, false)
+                    ];
+                case 1:
+                    _ref = _state.sent(), handle = _ref.handle, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
+                    store = dialect.createStore(handle, conn, resolvedCfg);
+                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+                    (0, _handoffts.rekeyChunkText)(conn, store);
+                    return [
+                        2,
+                        {
+                            store: store,
+                            cfg: resolvedCfg,
+                            dbPath: dbPath,
+                            parsed: parsed,
+                            warnings: warnings
+                        }
+                    ];
+            }
+        });
+    })();
+}
+function openWithBuilder(cfg, dialect) {
+    return _async_to_generator(function() {
+        var _ref, handle, conn, resolvedCfg, dbPath, parsed, warnings, builder, store;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        connectWithDialect(cfg, dialect, true)
+                    ];
+                case 1:
+                    _ref = _state.sent(), handle = _ref.handle, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings, builder = _ref.builder;
+                    store = dialect.createStore(handle, conn, resolvedCfg);
+                    (0, _handoffts.rekeyChunkText)(conn, store);
+                    return [
+                        2,
+                        {
+                            store: store,
+                            cfg: resolvedCfg,
+                            dbPath: dbPath,
+                            parsed: parsed,
+                            warnings: warnings,
+                            builder: builder
+                        }
+                    ];
+            }
+        });
+    })();
+}
+function docCount(store) {
+    return _async_to_generator(function() {
+        var stmt;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        store.prepare('SELECT COUNT(*) AS n FROM frontmatter')
+                    ];
+                case 1:
+                    stmt = _state.sent();
+                    return [
+                        4,
+                        stmt.get()
+                    ];
+                case 2:
+                    return [
+                        2,
+                        _state.sent().n
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/open.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../config/index.ts';\nimport { rekeyChunkText } from '../embed/handoff.ts';\nimport { SenseError } from '../errors.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { Builder } from './builder.ts';\nimport { createBuilder } from './builder.ts';\nimport { clearCache } from './cache.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport { classifyEmbedChange } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { classifyFeatureToggles, isFeatureOnlyChange } from './feature-scope.ts';\nimport { forcedPresetPaths, isPresetOnlyChange } from './preset-scope.ts';\nimport { getMeta, setMeta } from './shared.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, signatureDiff } from './signature.ts';\nimport type { Connection, OpenDialect, Store } from './types.ts';\n\n// One open algorithm shared by every store, parameterised by a per-engine OpenDialect (types.ts).\n// Ordering is universal, not a dialect concern: connect, meta table, schema-version and\n// feature-signature comparison (rebuild, adopt, or partial rebuild), any extra rebuild trigger,\n// ensureSchema, the derived busy_timeout PRAGMA, reconcile, then the store itself.\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\ninterface ConnectResult<Handle> {\n  handle: Handle;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n  // Closed already unless `keepBuilderOpen` was set: a one-shot open reconciles once and is done\n  // with it, a watcher keeps calling build() on the same instance across its whole run.\n  builder: Builder;\n}\n\n// duckdb and turso hold the cache file for their connection's whole life, not for a transaction,\n// so a second command waits on the first command finishing. Bounded rather than open-ended: this\n// clears a warm query's collision (~100ms) and still fails loudly behind a cold build instead of\n// looking hung.\nconst LOCK_RETRY_MS = 5_000;\nconst LOCK_POLL_MS = 50;\n\nasync function connectUnlocked<Handle>(dbPath: string, cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<{ handle: Handle; conn: Connection }> {\n  const deadline = Date.now() + LOCK_RETRY_MS;\n  for (;;) {\n    try {\n      return await dialect.connect(dbPath, cfg);\n    } catch (err) {\n      if (!dialect.isLocked?.(err as Error)) throw err;\n      if (Date.now() >= deadline) {\n        throw new SenseError('STORE_BUSY', `another sense process is using this tree's ${cfg.store} cache (${dbPath}) and did not release it within ${LOCK_RETRY_MS / 1000}s; wait for that command to finish, or set \"store\" to \"sqlite\" in sense.config.json, which serves concurrent commands`);\n      }\n      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));\n    }\n  }\n}\n\nasync function connectWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>, keepBuilderOpen: boolean): Promise<ConnectResult<Handle>> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, dialect.filename);\n\n  const { handle, conn } = await connectUnlocked(dbPath, cfg, dialect);\n  // A throw below must release this handle, or the leaked WAL makes the cache file undeletable on\n  // Windows. `closed` keeps the catch from double-closing a handle a rebuild branch already closed.\n  let closed = false;\n  // Created here, not inside reconcile(): its pool must survive across every build() call this\n  // connection sees (a watcher's repeated ticks), not just the one below.\n  const builder = createBuilder(conn, cfg, cfg.baseDir, dialect.reconcileDialect);\n  try {\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    // Set only by the preset-membership branch below; reconcile treats these paths as touched\n    // despite an unchanged stamp, for this one build() call.\n    let forcedPaths: Set<string> | undefined;\n    // Set only by the embed-narrow branch below; applied via builder.invalidate() after ensureSchema.\n    let embedInvalidate: EmbedChangeKind | undefined;\n    // Set only by the feature-toggle branch below; applied via builder.invalidateFeatures() after ensureSchema.\n    let featureToggles: FeatureToggle[] | undefined;\n    if ((version !== null && version !== dialect.schemaVersion) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== dialect.schemaVersion) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        // Close before clearCache deletes the files out from under the still-open handle.\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Null (rather than the empty set) when changedKeys isn't preset-only, or a changed\n      // preset's stored segment can't be parsed back into old include/exclude/vectors -- either\n      // way the branch below is skipped and the full rebuild stays the fallback.\n      const presetForced = isPresetOnlyChange(changedKeys) ? forcedPresetPaths(cfg, cfg.baseDir, features ?? '', changedKeys) : null;\n      // Null unless exactly one segment changed and it decomposes into a recognised embed-only\n      // case (embed-scope.ts): anything else, including embed toggled on/off, stays null.\n      const embedKind = changedKeys.size === 1 && changedKeys.has('embed') ? classifyEmbedChange(features ?? '', wantFeatures) : null;\n      // Null (rather than the empty set) when changedKeys isn't feature-toggle-only, or a changed\n      // feature's stored segment can't be parsed back into old/new on-off -- either way the branch\n      // below is skipped and the full rebuild stays the fallback.\n      const toggles = isFeatureOnlyChange(changedKeys) ? classifyFeatureToggles(features ?? '', wantFeatures, changedKeys) : null;\n      if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (embedKind !== null) {\n        // Chunk boundaries are file-derived, not model-derived (embed.ts's chunksOf), so neither\n        // case needs any file reparsed for any feature but embed itself.\n        embedInvalidate = embedKind;\n        console.error(embedKind === 'model' ? 'sense: config change (embed settings) invalidates only the stale vectors' : 'sense: config change (embed settings) rebuilds only the embeddings it affects');\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (presetForced !== null) {\n        // Membership only: which files a preset covers moved, not a file's own content, a\n        // global feature, or the embed model. reconcile() already knows how to\n        // add, update, and remove a file correctly (every cross-feature cascade included); it\n        // just needs telling which unchanged files to treat as touched (forcedPaths, above).\n        forcedPaths = presetForced;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the files it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else if (toggles !== null) {\n        // A feature turned off or on: reconcile.ts's activeFeatures skips a disabled feature's\n        // hooks entirely, so its rows rot while off -- invalidateFeatures drops them and, for a\n        // feature turning back on, fully re-derives across every indexed file rather than\n        // trusting rows written before or during that window.\n        featureToggles = toggles;\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) reparses only the feature it affects`);\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        await dialect.close(handle);\n        clearCache(cfg);\n        return connectWithDialect(cfg, dialect, keepBuilderOpen);\n      }\n    }\n\n    await dialect.ensureSchema(handle, conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before build() -- that call is the one that races a watcher's transaction.\n    if (dialect.setDerivedBusyTimeout) {\n      const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n      await dialect.setDerivedBusyTimeout(handle, conn, Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000));\n    }\n\n    // Runs before build(): forcedPaths, embedInvalidate and featureToggles never target the same\n    // call (each comes from its own mutually exclusive branch above: a preset-only, embed-only, or\n    // feature-toggle-only changed-key set).\n    const embedParsed = embedInvalidate ? (await builder.invalidate(embedInvalidate)).parsed : 0;\n    const featureParsed = featureToggles ? (await builder.invalidateFeatures(featureToggles)).parsed : 0;\n\n    const { parsed, warnings } = await builder.build(forcedPaths);\n    // A one-shot open is done reconciling for good; a watcher keeps `builder` alive across its run.\n    if (!keepBuilderOpen) await builder.close();\n\n    return { handle, conn, cfg, dbPath, parsed: parsed + embedParsed + featureParsed, warnings, builder };\n  } catch (err) {\n    await builder.close();\n    if (!closed) await dialect.close(handle);\n    throw err;\n  }\n}\n\n// A store's open(): connects (see connectWithDialect above), then wraps the resulting connection\n// in the Store interface. The builder's pool closes right after the initial build.\nexport async function openWithDialect<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connectWithDialect(cfg, dialect, false);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\n// Same connect+build as openWithDialect, but keeps the builder's pool alive and hands it back\n// alongside the store, for a caller (a watcher) that reconciles repeatedly on this connection.\nexport async function openWithBuilder<Handle>(cfg: ResolvedConfig, dialect: OpenDialect<Handle>): Promise<OpenResult & { builder: Builder }> {\n  const { handle, conn, cfg: resolvedCfg, dbPath, parsed, warnings, builder } = await connectWithDialect(cfg, dialect, true);\n  const store = dialect.createStore(handle, conn, resolvedCfg);\n  rekeyChunkText(conn, store);\n  return { store, cfg: resolvedCfg, dbPath, parsed, warnings, builder };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["docCount","openWithBuilder","openWithDialect","LOCK_RETRY_MS","LOCK_POLL_MS","connectUnlocked","dbPath","cfg","dialect","deadline","err","Date","now","connect","isLocked","SenseError","store","Promise","resolve","setTimeout","connectWithDialect","keepBuilderOpen","stateDir","handle","conn","closed","builder","version","features","wantFeatures","forcedPaths","embedInvalidate","featureToggles","changedKeys","presetForced","embedKind","toggles","changed","recordedMaxMs","embedParsed","featureParsed","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","filename","createBuilder","reconcileDialect","exec","getMeta","featureSignature","FEATURES","schemaVersion","console","error","close","clearCache","changedSignatureKeys","isPresetOnlyChange","forcedPresetPaths","size","has","classifyEmbedChange","isFeatureOnlyChange","classifyFeatureToggles","embedIdentityAdopted","setMeta","signatureDiff","ensureSchema","setDerivedBusyTimeout","Number","Math","min","max","invalidate","invalidateFeatures","build","resolvedCfg","createStore","rekeyChunkText","stmt","prepare","get","n"],"mappings":";;;;;;;;;;;QAyMsBA;eAAAA;;QAPAC;eAAAA;;QAVAC;eAAAA;;;sBAxLI;wBACL;uBAEuB;yBACb;wBACJ;wBACF;yBAEK;uBACH;4BAES;8BAEwB;6BACN;wBACrB;2BACyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4B1E,iGAAiG;AACjG,iGAAiG;AACjG,iGAAiG;AACjG,gBAAgB;AAChB,IAAMC,gBAAgB;AACtB,IAAMC,eAAe;AAErB,SAAeC,gBAAwBC,MAAc,EAAEC,GAAmB,EAAEC,OAA4B;;YAChGC,UAIKC,KACFF;;;;oBALHC,WAAWE,KAAKC,GAAG,KAAKT;;;;;;;;;oBAGnB;;wBAAMK,QAAQK,OAAO,CAACP,QAAQC;;;oBAArC;;wBAAO;;;oBACAG;oBACP,IAAI,GAACF,oBAAAA,QAAQM,QAAQ,cAAhBN,wCAAAA,uBAAAA,SAAmBE,OAAe,MAAMA;oBAC7C,IAAIC,KAAKC,GAAG,MAAMH,UAAU;wBAC1B,MAAM,IAAIM,oBAAU,CAAC,cAAc,AAAC,8CAAiET,OAApBC,IAAIS,KAAK,EAAC,YAAmDb,OAAzCG,QAAO,oCAAuD,OAArBH,gBAAgB,MAAK;oBACrK;oBACA;;wBAAM,IAAIc,QAAQ,SAACC;mCAAYC,WAAWD,SAASd;;;;oBAAnD;;;;;;;;;;;;;;;;IAGN;;AAEA,SAAegB,mBAA2Bb,GAAmB,EAAEC,OAA4B,EAAEa,eAAwB;;YAC7GC,UAEAhB,QAEmB,MAAjBiB,QAAQC,MAGZC,QAGEC,SAMEC,SACAC,UACAC,cAGFC,aAEAC,iBAEAC,gBAYIC,aAIAC,cAGAC,WAIAC,SAkBEC,SASAA,UAIAA,UAcsB,OAAxBC,eAOFC,mBACAC,sBAEuB,OAArBC,QAAQC,UAKThC;;;;oBA5GHY,WAAWqB,IAAAA,cAAI,EAACpC,IAAIqC,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACxB,UAAU;wBAAEyB,WAAW;oBAAK;oBAChCzC,SAASqC,IAAAA,cAAI,EAACrB,UAAUd,QAAQwC,QAAQ;oBAErB;;wBAAM3C,gBAAgBC,QAAQC,KAAKC;;;oBAAnC,OAAA,eAAjBe,SAAiB,KAAjBA,QAAQC,OAAS,KAATA;oBAChB,gGAAgG;oBAChG,kGAAkG;oBAC9FC,SAAS;oBACb,6FAA6F;oBAC7F,wEAAwE;oBAClEC,UAAUuB,IAAAA,wBAAa,EAACzB,MAAMjB,KAAKA,IAAIqC,OAAO,EAAEpC,QAAQ0C,gBAAgB;;;;;;;;;oBAE5E;;wBAAM1B,KAAK2B,IAAI,CAAC;;;oBAAhB;oBAIgB;;wBAAMC,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA9BG,UAAU;oBACC;;wBAAMyB,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA/BI,WAAW;oBACXC,eAAewB,IAAAA,yBAAgB,EAAC9C,KAAK+C,kBAAQ;yBAQ/C,CAAA,AAAC3B,YAAY,QAAQA,YAAYnB,QAAQ+C,aAAa,IAAM3B,aAAa,QAAQA,aAAaC,YAAY,GAA1G;;;;yBAGEF,CAAAA,YAAY,QAAQA,YAAYnB,QAAQ+C,aAAa,AAAD,GAApD5B;;;;oBACF6B,QAAQC,KAAK,CAAC;oBACd,kFAAkF;oBAClFhC,SAAS;oBACT;;wBAAMjB,QAAQkD,KAAK,CAACnC;;;oBAApB;oBACAoC,IAAAA,mBAAU,EAACpD;oBACX;;wBAAOa,mBAAmBb,KAAKC,SAASa;;;oBAEpCY,cAAc2B,IAAAA,iCAAoB,EAAChC,qBAAAA,sBAAAA,WAAY,IAAIC;oBACzD,oFAAoF;oBACpF,0FAA0F;oBAC1F,2EAA2E;oBACrEK,eAAe2B,IAAAA,iCAAkB,EAAC5B,eAAe6B,IAAAA,gCAAiB,EAACvD,KAAKA,IAAIqC,OAAO,EAAEhB,qBAAAA,sBAAAA,WAAY,IAAIK,eAAe;oBAC1H,yFAAyF;oBACzF,oFAAoF;oBAC9EE,YAAYF,YAAY8B,IAAI,KAAK,KAAK9B,YAAY+B,GAAG,CAAC,WAAWC,IAAAA,iCAAmB,EAACrC,qBAAAA,sBAAAA,WAAY,IAAIC,gBAAgB;oBAC3H,4FAA4F;oBAC5F,6FAA6F;oBAC7F,4DAA4D;oBACtDO,UAAU8B,IAAAA,mCAAmB,EAACjC,eAAekC,IAAAA,sCAAsB,EAACvC,qBAAAA,sBAAAA,WAAY,IAAIC,cAAcI,eAAe;yBACnHA,CAAAA,YAAY8B,IAAI,KAAK,KAAK9B,YAAY+B,GAAG,CAAC,YAAYI,IAAAA,iCAAoB,EAACxC,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGI;;;;oBACF,iFAAiF;oBACjF,sDAAsD;oBACtDuB,QAAQC,KAAK,CAAC;oBACd;;wBAAMY,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSM,CAAAA,cAAc,IAAG,GAAjBA;;;;oBACT,yFAAyF;oBACzF,iEAAiE;oBACjEJ,kBAAkBI;oBAClBqB,QAAQC,KAAK,CAACtB,cAAc,UAAU,6EAA6E;oBACnH;;wBAAMkC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSK,CAAAA,iBAAiB,IAAG,GAApBA;;;;oBACT,kFAAkF;oBAClF,uEAAuE;oBACvE,sFAAsF;oBACtF,qFAAqF;oBACrFJ,cAAcI;oBACRG,UAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,SAAQ;oBAC/C;;wBAAMgC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;yBACSO,CAAAA,YAAY,IAAG,GAAfA;;;;oBACT,uFAAuF;oBACvF,wFAAwF;oBACxF,kFAAkF;oBAClF,sDAAsD;oBACtDJ,iBAAiBI;oBACXC,WAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,UAAQ;oBAC/C;;wBAAMgC,IAAAA,iBAAO,EAAC7C,MAAM,YAAYK;;;oBAAhC;;;;;;oBAEMQ,WAAUiC,IAAAA,0BAAa,EAAC1C,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9C2B,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARpB,UAAQ;oBAC/CZ,SAAS;oBACT;;wBAAMjB,QAAQkD,KAAK,CAACnC;;;oBAApB;oBACAoC,IAAAA,mBAAU,EAACpD;oBACX;;wBAAOa,mBAAmBb,KAAKC,SAASa;;;oBAI5C;;wBAAMb,QAAQ+D,YAAY,CAAChD,QAAQC,MAAMjB;;;oBAAzC;yBAIIC,QAAQgE,qBAAqB,EAA7BhE;;;;oBAC4B;;wBAAM4C,IAAAA,iBAAO,EAAC5B,MAAM;;;oBAA5Cc,gBAAgBmC;yBAAQ,QAAA,2BAAA,mBAAA,QAA4C;;oBAC1E;;wBAAMjE,QAAQgE,qBAAqB,CAACjD,QAAQC,MAAMkD,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItC,gBAAgB;;;oBAA/F;;;yBAMkBP,iBAAAA;;;;oBAAmB;;wBAAML,QAAQmD,UAAU,CAAC9C;;;2BAA1B,AAAC,cAA2CU,MAAM;;;;;;2BAAG;;;oBAArFF;yBACgBP,gBAAAA;;;;oBAAkB;;wBAAMN,QAAQoD,kBAAkB,CAAC9C;;;4BAAlC,AAAC,cAAkDS,MAAM;;;;;;4BAAG;;;oBAA7FD;oBAEuB;;wBAAMd,QAAQqD,KAAK,CAACjD;;;oBAApB,QAAA,eAArBW,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;yBAEZ,CAACrB,iBAAD;;;;oBAAkB;;wBAAMK,QAAQgC,KAAK;;;oBAAnB;;;oBAEtB;;wBAAO;4BAAEnC,QAAAA;4BAAQC,MAAAA;4BAAMjB,KAAAA;4BAAKD,QAAAA;4BAAQmC,QAAQA,SAASF,cAAcC;4BAAeE,UAAAA;4BAAUhB,SAAAA;wBAAQ;;;oBAC7FhB;oBACP;;wBAAMgB,QAAQgC,KAAK;;;oBAAnB;yBACI,CAACjC,QAAD;;;;oBAAS;;wBAAMjB,QAAQkD,KAAK,CAACnC;;;oBAApB;;;oBACb,MAAMb;;;;;;;IAEV;;AAIO,SAAeR,gBAAwBK,GAAmB,EAAEC,OAA4B;;YACxB,MAA7De,QAAQC,MAAWwD,aAAa1E,QAAQmC,QAAQC,UAClD1B;;;;oBAD+D;;wBAAMI,mBAAmBb,KAAKC,SAAS;;;oBAAvC,OAAA,eAA7De,SAA6D,KAA7DA,QAAQC,OAAqD,KAArDA,MAAWwD,cAA0C,KAA/CzE,KAAkBD,SAA6B,KAA7BA,QAAQmC,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAClD1B,QAAQR,QAAQyE,WAAW,CAAC1D,QAAQC,MAAMwD;oBAChD,0FAA0F;oBAC1FE,IAAAA,yBAAc,EAAC1D,MAAMR;oBACrB;;wBAAO;4BAAEA,OAAAA;4BAAOT,KAAKyE;4BAAa1E,QAAAA;4BAAQmC,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IAC7D;;AAIO,SAAezC,gBAAwBM,GAAmB,EAAEC,OAA4B;;YACf,MAAtEe,QAAQC,MAAWwD,aAAa1E,QAAQmC,QAAQC,UAAUhB,SAC5DV;;;;oBADwE;;wBAAMI,mBAAmBb,KAAKC,SAAS;;;oBAAvC,OAAA,eAAtEe,SAAsE,KAAtEA,QAAQC,OAA8D,KAA9DA,MAAWwD,cAAmD,KAAxDzE,KAAkBD,SAAsC,KAAtCA,QAAQmC,SAA8B,KAA9BA,QAAQC,WAAsB,KAAtBA,UAAUhB,UAAY,KAAZA;oBAC5DV,QAAQR,QAAQyE,WAAW,CAAC1D,QAAQC,MAAMwD;oBAChDE,IAAAA,yBAAc,EAAC1D,MAAMR;oBACrB;;wBAAO;4BAAEA,OAAAA;4BAAOT,KAAKyE;4BAAa1E,QAAAA;4BAAQmC,QAAAA;4BAAQC,UAAAA;4BAAUhB,SAAAA;wBAAQ;;;;IACtE;;AAEO,SAAe1B,SAASgB,KAAY;;YACnCmE;;;;oBAAO;;wBAAMnE,MAAMoE,OAAO,CAAC;;;oBAA3BD,OAAO;oBACJ;;wBAAMA,KAAKE,GAAG;;;oBAAvB;;wBAAS,cAAoCC,CAAC;;;;IAChD"}

package/dist/cjs/store/preset-scope.d.cts

@@ -0,0 +1,3 @@
+import type { Config } from '../config/index.js';
+export declare function isPresetOnlyChange(changedKeys: ReadonlySet<string>): boolean;
+export declare function forcedPresetPaths(cfg: Config, baseDir: string, before: string, changedKeys: ReadonlySet<string>): Set<string> | null;

package/dist/cjs/store/preset-scope.d.ts

@@ -0,0 +1,3 @@
+import type { Config } from '../config/index.js';
+export declare function isPresetOnlyChange(changedKeys: ReadonlySet<string>): boolean;
+export declare function forcedPresetPaths(cfg: Config, baseDir: string, before: string, changedKeys: ReadonlySet<string>): Set<string> | null;