sensemaking 0.19.2 → 0.21.0

package/dist/cjs/store/builder.js

@@ -0,0 +1,618 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "createBuilder", {
+    enumerable: true,
+    get: function() {
+        return createBuilder;
+    }
+});
+var _embedts = require("../features/embed.js");
+var _indexts = require("../features/index.js");
+var _rankts = require("../features/rank.js");
+var _indexts1 = require("../scan/index.js");
+var _poolts = require("../scan/pool.js");
+var _reparsets = require("../scan/reparse.js");
+var _featurescopets = require("./feature-scope.js");
+var _reconcilets = require("./reconcile.js");
+var _sharedts = require("./shared.js");
+var _transactionts = require("./transaction.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+// 'model': the provider or model name moved but chunk boundaries did not, so only the vector
+// values are stale. No reparse, and no table but embeddings is touched.
+function nullEmbedVectors(conn, dialect) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL');
+                        }, dialect.beginMode())
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: 0
+                        }
+                    ];
+            }
+        });
+    })();
+}
+// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds
+// every embedding row through the embed feature's own remove/store, touching no other table.
+function rebuildEmbeddings(conn, cfg, baseDir, dialect, pool) {
+    return _async_to_generator(function() {
+        var existingStmt, existingPaths, _, files, docs, paths, delta;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.prepare('SELECT "path" FROM frontmatter')
+                    ];
+                case 1:
+                    existingStmt = _state.sent();
+                    _ = Set.bind;
+                    return [
+                        4,
+                        existingStmt.all()
+                    ];
+                case 2:
+                    existingPaths = new (_.apply(Set, [
+                        void 0,
+                        _state.sent().map(function(r) {
+                            return r.path;
+                        })
+                    ]));
+                    // A file added since the last build has no row yet; leaving it for build() to insert avoids a
+                    // primary-key collision between this INSERT and build()'s own insert for the same new file.
+                    files = (0, _indexts1.listFiles)(cfg, baseDir).filter(function(f) {
+                        return f.embed && existingPaths.has(f.relPath);
+                    });
+                    if (files.length === 0) return [
+                        2,
+                        {
+                            parsed: 0
+                        }
+                    ];
+                    return [
+                        4,
+                        (0, _reparsets.reparseFiles)(files, [
+                            _embedts.embed
+                        ], cfg, new Set(), undefined, {
+                            pool: pool
+                        })
+                    ];
+                case 3:
+                    docs = _state.sent().docs;
+                    paths = docs.map(function(d) {
+                        return d.relPath;
+                    });
+                    delta = {
+                        files: files,
+                        reparsed: paths,
+                        added: [],
+                        vanished: []
+                    };
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var _embed_remove, _embed_store;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                (_embed_remove = _embedts.embed.remove) === null || _embed_remove === void 0 ? void 0 : _embed_remove.call(_embedts.embed, conn, paths, delta)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (_embed_store = _embedts.embed.store) === null || _embed_store === void 0 ? void 0 : _embed_store.call(_embedts.embed, conn, docs.map(function(d) {
+                                                    return {
+                                                        path: d.relPath,
+                                                        extracted: d.extracted.embed
+                                                    };
+                                                }), delta)
+                                            ];
+                                        case 2:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, dialect.beginMode())
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: docs.length
+                        }
+                    ];
+            }
+        });
+    })();
+}
+// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.
+// Rows left over from before this feature was disabled are untrustworthy by construction (files
+// added, edited, or deleted got no remove/store for it), so this never diffs against them --
+// dropping every row and reparsing the whole tree is the only safe path back to consistent state.
+function invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle) {
+    return _async_to_generator(function() {
+        var table, feature, files, docs, docsForFeature, paths, delta;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    table = _featurescopets.NARROW_FEATURE_TABLE[toggle.name];
+                    feature = _indexts.FEATURES.find(function(f) {
+                        return f.name === toggle.name;
+                    });
+                    if (!feature) throw new Error('invalidateFeatureTable: no registered feature named "'.concat(toggle.name, '"'));
+                    if (!!toggle.turnedOn) return [
+                        3,
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return conn.exec("DELETE FROM ".concat(table));
+                        }, dialect.beginMode())
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: 0
+                        }
+                    ];
+                case 2:
+                    files = (0, _indexts1.listFiles)(cfg, baseDir);
+                    return [
+                        4,
+                        (0, _reparsets.reparseFiles)(files, [
+                            feature
+                        ], cfg, new Set(), undefined, {
+                            pool: pool
+                        })
+                    ];
+                case 3:
+                    docs = _state.sent().docs;
+                    docsForFeature = docs.map(function(d) {
+                        return {
+                            path: d.relPath,
+                            extracted: d.extracted[feature.name]
+                        };
+                    });
+                    paths = docs.map(function(d) {
+                        return d.relPath;
+                    });
+                    // Every path counts as added, not just touched: the table was just dropped, so there is nothing
+                    // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path
+                    // only when added.length equals files.length.
+                    delta = {
+                        files: files,
+                        reparsed: paths,
+                        added: paths,
+                        vanished: []
+                    };
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var _feature_store, _feature_afterReconcile;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                conn.exec("DELETE FROM ".concat(table))
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta)
+                                            ];
+                                        case 2:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta)
+                                            ];
+                                        case 3:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, dialect.beginMode())
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: docs.length
+                        }
+                    ];
+            }
+        });
+    })();
+}
+// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column
+// presence, since links can toggle off while rank has never been enabled for this tree.
+function nullRank(conn, dialect) {
+    return _async_to_generator(function() {
+        var columns;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        (0, _sharedts.getColumns)(conn)
+                    ];
+                case 1:
+                    columns = _state.sent();
+                    if (!columns.has('_rank')) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return conn.exec('UPDATE frontmatter SET "_rank" = NULL');
+                        }, dialect.beginMode())
+                    ];
+                case 2:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so
+// turning rank back on needs no reparseFiles pass at all.
+function rerunRank(conn, dialect) {
+    return _async_to_generator(function() {
+        var delta;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    delta = {
+                        files: [],
+                        reparsed: [],
+                        added: [],
+                        vanished: []
+                    };
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var _rank_afterReconcile;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                (_rank_afterReconcile = _rankts.rank.afterReconcile) === null || _rank_afterReconcile === void 0 ? void 0 : _rank_afterReconcile.call(_rankts.rank, conn, delta)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, dialect.beginMode())
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles) {
+    return _async_to_generator(function() {
+        var byName, parsed, linksToggle, rankToggle, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, toggle, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    byName = new Map(toggles.map(function(toggle) {
+                        return [
+                            toggle.name,
+                            toggle
+                        ];
+                    }));
+                    parsed = 0;
+                    // links first, if present: rank's rerun below (whether from its own toggle or the co-change
+                    // rankToggle carries) must read the links table after it is back in its new state.
+                    linksToggle = byName.get('links');
+                    if (!linksToggle) return [
+                        3,
+                        3
+                    ];
+                    return [
+                        4,
+                        invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)
+                    ];
+                case 1:
+                    parsed = parsed + _state.sent().parsed;
+                    if (!!linksToggle.turnedOn) return [
+                        3,
+                        3
+                    ];
+                    return [
+                        4,
+                        nullRank(conn, dialect)
+                    ];
+                case 2:
+                    _state.sent();
+                    _state.label = 3;
+                case 3:
+                    // rank's effective on/off (featureEnabled's links dependency) only appears as a changed
+                    // `feature:rank` segment when it actually flips -- whether that's rank's own config or a
+                    // links toggle taking it along -- so honoring it here is complete on its own.
+                    rankToggle = byName.get('rank');
+                    if (!rankToggle) return [
+                        3,
+                        7
+                    ];
+                    if (!rankToggle.turnedOn) return [
+                        3,
+                        5
+                    ];
+                    return [
+                        4,
+                        rerunRank(conn, dialect)
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        3,
+                        7
+                    ];
+                case 5:
+                    return [
+                        4,
+                        nullRank(conn, dialect)
+                    ];
+                case 6:
+                    _state.sent();
+                    _state.label = 7;
+                case 7:
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 8;
+                case 8:
+                    _state.trys.push([
+                        8,
+                        13,
+                        14,
+                        15
+                    ]);
+                    _iterator = toggles[Symbol.iterator]();
+                    _state.label = 9;
+                case 9:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        12
+                    ];
+                    toggle = _step.value;
+                    if (toggle.name === 'links' || toggle.name === 'rank') return [
+                        3,
+                        11
+                    ];
+                    return [
+                        4,
+                        invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)
+                    ];
+                case 10:
+                    parsed = parsed + _state.sent().parsed;
+                    _state.label = 11;
+                case 11:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        9
+                    ];
+                case 12:
+                    return [
+                        3,
+                        15
+                    ];
+                case 13:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        15
+                    ];
+                case 14:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 15:
+                    return [
+                        2,
+                        {
+                            parsed: parsed
+                        }
+                    ];
+            }
+        });
+    })();
+}
+function createBuilder(conn, cfg, baseDir, dialect) {
+    var pool = new _poolts.ParsePool();
+    return {
+        build: function build(forcedPaths) {
+            return (0, _reconcilets.reconcile)(conn, cfg, baseDir, dialect, pool, forcedPaths);
+        },
+        invalidate: function invalidate(kind) {
+            return kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool);
+        },
+        invalidateFeatures: function invalidateFeatures(toggles) {
+            return invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles);
+        },
+        close: function close() {
+            return pool.close();
+        },
+        get poolsCreated () {
+            return pool.poolsCreated;
+        }
+    };
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/builder.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","nullEmbedVectors","conn","dialect","withTransaction","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","existingPaths","files","docs","paths","delta","prepare","Set","all","map","r","path","listFiles","filter","f","embed","has","relPath","length","reparseFiles","undefined","d","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","feature","docsForFeature","NARROW_FEATURE_TABLE","name","FEATURES","find","Error","turnedOn","afterReconcile","nullRank","columns","getColumns","rerunRank","rank","invalidateFeatureToggles","toggles","byName","linksToggle","rankToggle","Map","get","ParsePool","build","forcedPaths","reconcile","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":";;;;+BA6JgBA;;;eAAAA;;;uBA5JM;uBACG;sBACJ;wBAEK;sBACA;yBACG;8BAGQ;2BACX;wBACC;6BACK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBhC,6FAA6F;AAC7F,wEAAwE;AACxE,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;;;;;oBACzE;;wBAAMC,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAAsDF,QAAQG,SAAS;;;oBAAnH;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;;IACrB;;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,SAAeC,kBAAkBN,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe;;YACnHC,cACAC,kBAGAC,OAGEC,MACFC,OACAC;;;;oBATe;;wBAAMf,KAAKgB,OAAO,CAAC;;;oBAAlCN,eAAe;wBACKO;oBAAK;;wBAAMP,aAAaQ,GAAG;;;oBAA/CP,gBAAgB,IAAA,CAAA,EAAA,MAAIM;;wBAAK,cAA0BE,GAAG,CAAC,SAACC;mCAAM,AAACA,EAAuBC,IAAI;;sBAAC;oBACjG,8FAA8F;oBAC9F,4FAA4F;oBACtFT,QAAQU,IAAAA,mBAAS,EAACf,KAAKC,SAASe,MAAM,CAAC,SAACC;+BAAMA,EAAEC,KAAK,IAAId,cAAce,GAAG,CAACF,EAAEG,OAAO;;oBAC1F,IAAIf,MAAMgB,MAAM,KAAK,GAAG;;wBAAO;4BAAEvB,QAAQ;wBAAE;;oBAE1B;;wBAAMwB,IAAAA,uBAAY,EAACjB;4BAAQa,cAAK;2BAAGlB,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAA9EI,OAAS,cAATA;oBACFC,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACjCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,KAAK;wBAAMC,QAAQ;oBAAK;oBAChF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQyB,eACAA;;;;4CADN;;iDAAMA,gBAAAA,cAAK,CAACU,MAAM,cAAZV,oCAAAA,mBAAAA,cAAK,EAAUzB,MAAMc,OAAOC;;;4CAAlC;4CACA;;iDAAMU,eAAAA,cAAK,CAACW,KAAK,cAAXX,mCAAAA,kBAAAA,cAAK,EACTzB,MACAa,KAAKM,GAAG,CAAC,SAACY;2DAAO;wDAAEV,MAAMU,EAAEJ,OAAO;wDAAEU,WAAWN,EAAEM,SAAS,CAACZ,KAAK;oDAAC;oDACjEV;;;4CAHF;;;;;;4BAKF;2BACAd,QAAQG,SAAS;;;oBAVnB;oBAYA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,SAAeU,uBAAuBtC,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE8B,MAAqB;;YAC/IC,OACAC,SAQA7B,OACEC,MACF6B,gBACA5B,OAIAC;;;;oBAhBAyB,QAAQG,oCAAoB,CAACJ,OAAOK,IAAI,CAAC;oBACzCH,UAAUI,iBAAQ,CAACC,IAAI,CAAC,SAACtB;+BAAMA,EAAEoB,IAAI,KAAKL,OAAOK,IAAI;;oBAC3D,IAAI,CAACH,SAAS,MAAM,IAAIM,MAAM,AAAC,wDAAmE,OAAZR,OAAOK,IAAI,EAAC;yBAE9F,CAACL,OAAOS,QAAQ,EAAhB;;;;oBACF;;wBAAM9C,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;2BAAUvC,QAAQG,SAAS;;;oBAAtF;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;oBAGfO,QAAQU,IAAAA,mBAAS,EAACf,KAAKC;oBACZ;;wBAAMqB,IAAAA,uBAAY,EAACjB;4BAAQ6B;2BAAUlC,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAAhFI,OAAS,cAATA;oBACF6B,iBAAiB7B,KAAKM,GAAG,CAAC,SAACY;+BAAO;4BAAEV,MAAMU,EAAEJ,OAAO;4BAAEU,WAAWN,EAAEM,SAAS,CAACI,QAAQG,IAAI,CAAC;wBAAC;;oBAC1F9B,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACvC,gGAAgG;oBAChG,8FAA8F;oBAC9F,8CAA8C;oBACxCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,OAAOnB;wBAAOoB,QAAQ;oBAAK;oBACnF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCAEQyC,gBACAA;;;;4CAFN;;gDAAMzC,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;;;4CAA/B;4CACA;;iDAAMC,iBAAAA,QAAQL,KAAK,cAAbK,qCAAAA,oBAAAA,SAAgBzC,MAAM0C,gBAAgB3B;;;4CAA5C;4CACA;;iDAAM0B,0BAAAA,QAAQQ,cAAc,cAAtBR,8CAAAA,6BAAAA,SAAyBzC,MAAMe;;;4CAArC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBAPnB;oBASA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,SAAesB,SAASlD,IAAgB,EAAEC,OAAyB;;YAC3DkD;;;;oBAAU;;wBAAMC,IAAAA,oBAAU,EAACpD;;;oBAA3BmD,UAAU;oBAChB,IAAI,CAACA,QAAQzB,GAAG,CAAC,UAAU;;;oBAC3B;;wBAAMxB,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAA0CF,QAAQG,SAAS;;;oBAAvG;;;;;;IACF;;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAAeiD,UAAUrD,IAAgB,EAAEC,OAAyB;;YAC5Dc;;;;oBAAAA,QAAwB;wBAAEH,KAAK;wBAAMoB,QAAQ;wBAAMC,KAAK;wBAAMC,QAAQ;oBAAK;oBACjF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQsD;;;;4CAAN;;iDAAMA,uBAAAA,YAAI,CAACL,cAAc,cAAnBK,2CAAAA,0BAAAA,YAAI,EAAkBtD,MAAMe;;;4CAAlC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBALnB;;;;;;IAOF;;AAEA,SAAemD,yBAAyBvD,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE+C,OAAwB;;YACpJC,QACFpD,QAIEqD,aAWAC,YAMD,2BAAA,mBAAA,gBAAA,WAAA,OAAMpB;;;;oBAtBLkB,SAAS,IAAIG,IAAIJ,QAAQrC,GAAG,CAAC,SAACoB;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL;yBAAO;;oBAChElC,SAAS;oBAEb,4FAA4F;oBAC5F,mFAAmF;oBAC7EqD,cAAcD,OAAOI,GAAG,CAAC;yBAC3BH,aAAAA;;;;oBACS;;wBAAMpB,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAMiD;;;oBAA3ErD,SAAAA,SAAU,AAAC,cAA8EA,MAAM;yBAG3F,CAACqD,YAAYV,QAAQ,EAArB;;;;oBAAuB;;wBAAME,SAASlD,MAAMC;;;oBAArB;;;oBAG7B,wFAAwF;oBACxF,yFAAyF;oBACzF,8EAA8E;oBACxE0D,aAAaF,OAAOI,GAAG,CAAC;yBAC1BF,YAAAA;;;;yBACEA,WAAWX,QAAQ,EAAnBW;;;;oBAAqB;;wBAAMN,UAAUrD,MAAMC;;;oBAAtB;;;;;;oBACpB;;wBAAMiD,SAASlD,MAAMC;;;oBAArB;;;oBAGF,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAgBuD;;;2BAAhB,6BAAA,QAAA;;;;oBAAMjB,SAAN;oBACH,IAAIA,OAAOK,IAAI,KAAK,WAAWL,OAAOK,IAAI,KAAK,QAAQ;;;;oBAC5C;;wBAAMN,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAM8B;;;oBAA3ElC,SAAAA,SAAU,AAAC,cAAyEA,MAAM;;;oBAFvF;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAIL;;wBAAO;4BAAEA,QAAAA;wBAAO;;;;IAClB;;AAIO,SAASP,cAAcE,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB;IACrG,IAAMQ,OAAO,IAAIqD,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA,MAAQC;mBAAgBC,IAAAA,sBAAS,EAACjE,MAAMO,KAAKC,SAASP,SAASQ,MAAMuD;;QACrEE,YAAY,SAAZA,WAAaC;mBAAUA,SAAS,UAAUpE,iBAAiBC,MAAMC,WAAWK,kBAAkBN,MAAMO,KAAKC,SAASP,SAASQ;;QAC3H2D,oBAAoB,SAApBA,mBAAqBZ;mBAAYD,yBAAyBvD,MAAMO,KAAKC,SAASP,SAASQ,MAAM+C;;QAC7Fa,OAAO,SAAPA;mBAAa5D,KAAK4D,KAAK;;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}

package/dist/cjs/store/duckdb/lexical.d.cts

@@ -2,6 +2,7 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
+export declare function markContentStale(conn: Connection): void;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;

package/dist/cjs/store/duckdb/lexical.d.ts

@@ -2,6 +2,7 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
+export declare function markContentStale(conn: Connection): void;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;