sensemaking 0.20.0 → 0.21.1

package/dist/cjs/store/builder.d.cts

@@ -1,10 +1,18 @@
 import type { Config } from '../config/index.js';
+import type { EmbedChangeKind } from './embed-scope.js';
+import type { FeatureToggle } from './feature-scope.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
-    build(): Promise<{
+    build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
     }>;
+    invalidate(kind: EmbedChangeKind): Promise<{
+        parsed: number;
+    }>;
+    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
+        parsed: number;
+    }>;
     close(): Promise<void>;
     readonly poolsCreated: number;
 }

package/dist/cjs/store/builder.d.ts

@@ -1,10 +1,18 @@
 import type { Config } from '../config/index.js';
+import type { EmbedChangeKind } from './embed-scope.js';
+import type { FeatureToggle } from './feature-scope.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
-    build(): Promise<{
+    build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
     }>;
+    invalidate(kind: EmbedChangeKind): Promise<{
+        parsed: number;
+    }>;
+    invalidateFeatures(toggles: FeatureToggle[]): Promise<{
+        parsed: number;
+    }>;
     close(): Promise<void>;
     readonly poolsCreated: number;
 }

package/dist/cjs/store/builder.js

@@ -8,13 +8,604 @@ Object.defineProperty(exports, "createBuilder", {
         return createBuilder;
     }
 });
+var _embedts = require("../features/embed.js");
+var _indexts = require("../features/index.js");
+var _rankts = require("../features/rank.js");
+var _indexts1 = require("../scan/index.js");
 var _poolts = require("../scan/pool.js");
+var _reparsets = require("../scan/reparse.js");
+var _featurescopets = require("./feature-scope.js");
 var _reconcilets = require("./reconcile.js");
+var _sharedts = require("./shared.js");
+var _transactionts = require("./transaction.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+// 'model': the provider or model name moved but chunk boundaries did not, so only the vector
+// values are stale. No reparse, and no table but embeddings is touched.
+function nullEmbedVectors(conn, dialect) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL');
+                        }, dialect.beginMode())
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: 0
+                        }
+                    ];
+            }
+        });
+    })();
+}
+// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds
+// every embedding row through the embed feature's own remove/store, touching no other table.
+function rebuildEmbeddings(conn, cfg, baseDir, dialect, pool) {
+    return _async_to_generator(function() {
+        var existingStmt, existingPaths, _, files, docs, paths, delta;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.prepare('SELECT "path" FROM frontmatter')
+                    ];
+                case 1:
+                    existingStmt = _state.sent();
+                    _ = Set.bind;
+                    return [
+                        4,
+                        existingStmt.all()
+                    ];
+                case 2:
+                    existingPaths = new (_.apply(Set, [
+                        void 0,
+                        _state.sent().map(function(r) {
+                            return r.path;
+                        })
+                    ]));
+                    // A file added since the last build has no row yet; leaving it for build() to insert avoids a
+                    // primary-key collision between this INSERT and build()'s own insert for the same new file.
+                    files = (0, _indexts1.listFiles)(cfg, baseDir).filter(function(f) {
+                        return f.embed && existingPaths.has(f.relPath);
+                    });
+                    if (files.length === 0) return [
+                        2,
+                        {
+                            parsed: 0
+                        }
+                    ];
+                    return [
+                        4,
+                        (0, _reparsets.reparseFiles)(files, [
+                            _embedts.embed
+                        ], cfg, new Set(), undefined, {
+                            pool: pool
+                        })
+                    ];
+                case 3:
+                    docs = _state.sent().docs;
+                    paths = docs.map(function(d) {
+                        return d.relPath;
+                    });
+                    delta = {
+                        files: files,
+                        reparsed: paths,
+                        added: [],
+                        vanished: []
+                    };
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var _embed_remove, _embed_store;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                (_embed_remove = _embedts.embed.remove) === null || _embed_remove === void 0 ? void 0 : _embed_remove.call(_embedts.embed, conn, paths, delta)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (_embed_store = _embedts.embed.store) === null || _embed_store === void 0 ? void 0 : _embed_store.call(_embedts.embed, conn, docs.map(function(d) {
+                                                    return {
+                                                        path: d.relPath,
+                                                        extracted: d.extracted.embed
+                                                    };
+                                                }), delta)
+                                            ];
+                                        case 2:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, dialect.beginMode())
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: docs.length
+                        }
+                    ];
+            }
+        });
+    })();
+}
+// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.
+// Rows left over from before this feature was disabled are untrustworthy by construction (files
+// added, edited, or deleted got no remove/store for it), so this never diffs against them --
+// dropping every row and reparsing the whole tree is the only safe path back to consistent state.
+function invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle) {
+    return _async_to_generator(function() {
+        var table, feature, files, docs, docsForFeature, paths, delta;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    table = _featurescopets.NARROW_FEATURE_TABLE[toggle.name];
+                    feature = _indexts.FEATURES.find(function(f) {
+                        return f.name === toggle.name;
+                    });
+                    if (!feature) throw new Error('invalidateFeatureTable: no registered feature named "'.concat(toggle.name, '"'));
+                    if (!!toggle.turnedOn) return [
+                        3,
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return conn.exec("DELETE FROM ".concat(table));
+                        }, dialect.beginMode())
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: 0
+                        }
+                    ];
+                case 2:
+                    files = (0, _indexts1.listFiles)(cfg, baseDir);
+                    return [
+                        4,
+                        (0, _reparsets.reparseFiles)(files, [
+                            feature
+                        ], cfg, new Set(), undefined, {
+                            pool: pool
+                        })
+                    ];
+                case 3:
+                    docs = _state.sent().docs;
+                    docsForFeature = docs.map(function(d) {
+                        return {
+                            path: d.relPath,
+                            extracted: d.extracted[feature.name]
+                        };
+                    });
+                    paths = docs.map(function(d) {
+                        return d.relPath;
+                    });
+                    // Every path counts as added, not just touched: the table was just dropped, so there is nothing
+                    // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path
+                    // only when added.length equals files.length.
+                    delta = {
+                        files: files,
+                        reparsed: paths,
+                        added: paths,
+                        vanished: []
+                    };
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var _feature_store, _feature_afterReconcile;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                conn.exec("DELETE FROM ".concat(table))
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta)
+                                            ];
+                                        case 2:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta)
+                                            ];
+                                        case 3:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, dialect.beginMode())
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        2,
+                        {
+                            parsed: docs.length
+                        }
+                    ];
+            }
+        });
+    })();
+}
+// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column
+// presence, since links can toggle off while rank has never been enabled for this tree.
+function nullRank(conn, dialect) {
+    return _async_to_generator(function() {
+        var columns;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        (0, _sharedts.getColumns)(conn)
+                    ];
+                case 1:
+                    columns = _state.sent();
+                    if (!columns.has('_rank')) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return conn.exec('UPDATE frontmatter SET "_rank" = NULL');
+                        }, dialect.beginMode())
+                    ];
+                case 2:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so
+// turning rank back on needs no reparseFiles pass at all.
+function rerunRank(conn, dialect) {
+    return _async_to_generator(function() {
+        var delta;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    delta = {
+                        files: [],
+                        reparsed: [],
+                        added: [],
+                        vanished: []
+                    };
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var _rank_afterReconcile;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                (_rank_afterReconcile = _rankts.rank.afterReconcile) === null || _rank_afterReconcile === void 0 ? void 0 : _rank_afterReconcile.call(_rankts.rank, conn, delta)
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        }, dialect.beginMode())
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles) {
+    return _async_to_generator(function() {
+        var byName, parsed, linksToggle, rankToggle, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, toggle, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    byName = new Map(toggles.map(function(toggle) {
+                        return [
+                            toggle.name,
+                            toggle
+                        ];
+                    }));
+                    parsed = 0;
+                    // links first, if present: rank's rerun below (whether from its own toggle or the co-change
+                    // rankToggle carries) must read the links table after it is back in its new state.
+                    linksToggle = byName.get('links');
+                    if (!linksToggle) return [
+                        3,
+                        3
+                    ];
+                    return [
+                        4,
+                        invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)
+                    ];
+                case 1:
+                    parsed = parsed + _state.sent().parsed;
+                    if (!!linksToggle.turnedOn) return [
+                        3,
+                        3
+                    ];
+                    return [
+                        4,
+                        nullRank(conn, dialect)
+                    ];
+                case 2:
+                    _state.sent();
+                    _state.label = 3;
+                case 3:
+                    // rank's effective on/off (featureEnabled's links dependency) only appears as a changed
+                    // `feature:rank` segment when it actually flips -- whether that's rank's own config or a
+                    // links toggle taking it along -- so honoring it here is complete on its own.
+                    rankToggle = byName.get('rank');
+                    if (!rankToggle) return [
+                        3,
+                        7
+                    ];
+                    if (!rankToggle.turnedOn) return [
+                        3,
+                        5
+                    ];
+                    return [
+                        4,
+                        rerunRank(conn, dialect)
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        3,
+                        7
+                    ];
+                case 5:
+                    return [
+                        4,
+                        nullRank(conn, dialect)
+                    ];
+                case 6:
+                    _state.sent();
+                    _state.label = 7;
+                case 7:
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 8;
+                case 8:
+                    _state.trys.push([
+                        8,
+                        13,
+                        14,
+                        15
+                    ]);
+                    _iterator = toggles[Symbol.iterator]();
+                    _state.label = 9;
+                case 9:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        12
+                    ];
+                    toggle = _step.value;
+                    if (toggle.name === 'links' || toggle.name === 'rank') return [
+                        3,
+                        11
+                    ];
+                    return [
+                        4,
+                        invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)
+                    ];
+                case 10:
+                    parsed = parsed + _state.sent().parsed;
+                    _state.label = 11;
+                case 11:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        9
+                    ];
+                case 12:
+                    return [
+                        3,
+                        15
+                    ];
+                case 13:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        15
+                    ];
+                case 14:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 15:
+                    return [
+                        2,
+                        {
+                            parsed: parsed
+                        }
+                    ];
+            }
+        });
+    })();
+}
 function createBuilder(conn, cfg, baseDir, dialect) {
     var pool = new _poolts.ParsePool();
     return {
-        build: function build() {
-            return (0, _reconcilets.reconcile)(conn, cfg, baseDir, dialect, pool);
+        build: function build(forcedPaths) {
+            return (0, _reconcilets.reconcile)(conn, cfg, baseDir, dialect, pool, forcedPaths);
+        },
+        invalidate: function invalidate(kind) {
+            return kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool);
+        },
+        invalidateFeatures: function invalidateFeatures(toggles) {
+            return invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles);
         },
         close: function close() {
             return pool.close();