sensemaking 0.19.0 → 0.19.1

package/dist/cjs/store/turso/reconcile.js

@@ -16,6 +16,7 @@ var _reparsets = require("../../scan/reparse.js");
 var _segmentts = require("../../text/segment.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
+var _opents = require("./open.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -205,6 +206,8 @@ function ngramSidecar(text) {
 // Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with
 // no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.
 var MAX_FRONTMATTER_COLUMNS = 2000;
+// Changed files above which rebuilding the FTS index beats maintaining it per row (see reconcile).
+var FTS_REBUILD_THRESHOLD = 250;
 // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
 var INSERT_CONTENT_SQL = 'INSERT INTO content ("path", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)';
 function contentRow(doc) {
@@ -341,7 +344,7 @@ function reconcile(conn, cfg, baseDir) {
                         4,
                         (0, _transactionts.withTransaction)(conn, function() {
                             return _async_to_generator(function() {
-                                var _feature_remove, _feature_afterReconcile, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, err, rows, contentTouched, presetTouched, presetRows, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator1, _step1, doc, _iterator2, _step2, presetName, removedPaths, _iteratorNormalCompletion3, _didIteratorError3, _iteratorError3, _iterator3, _step3, feature, err, _iteratorNormalCompletion4, _didIteratorError4, _iteratorError4, _loop, _iterator4, _step4, err, _iteratorNormalCompletion5, _didIteratorError5, _iteratorError5, _iterator5, _step5, feature1, err;
+                                var _feature_remove, _feature_afterReconcile, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, err, rows, churn, bulk, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, name, err, contentTouched, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, ddl, err, presetTouched, presetRows, _iteratorNormalCompletion3, _didIteratorError3, _iteratorError3, _iteratorNormalCompletion4, _didIteratorError4, _iteratorError4, _iterator3, _step3, doc, _iterator4, _step4, presetName, removedPaths, _iteratorNormalCompletion5, _didIteratorError5, _iteratorError5, _iterator5, _step5, feature, err, _iteratorNormalCompletion6, _didIteratorError6, _iteratorError6, _loop, _iterator6, _step6, err, _iteratorNormalCompletion7, _didIteratorError7, _iteratorError7, _iterator7, _step7, feature1, err;
                                 return _ts_generator(this, function(_state) {
                                     switch(_state.label){
                                         case 0:
@@ -426,12 +429,82 @@ function reconcile(conn, cfg, baseDir) {
                                             _state.sent();
                                             _state.label = 10;
                                         case 10:
+                                            // Tantivy maintains the FTS index per inserted row and the cost grows with the batch, so a
+                                            // large insert is superlinear: 6,566 notes took 945s with the index live, 3.3s built after.
+                                            // Measured crossover against a full rebuild, in changed files: ~250 on a 6,566-note tree
+                                            // (100 -> 895ms, 300 -> 4.1s, 1,300 -> 45.5s, rebuild 3.35s), ~295 on 13,132 (250 -> 5.2s,
+                                            // 400 -> 10.0s, rebuild 6.7s). Doubling the corpus moved it 250 to 295, so a constant fits
+                                            // where a ratio does not: 20% would allow 1,313 files here, costing 45s to save 3.3s.
+                                            // Erring low is the cheap direction, since a needless rebuild is linear and predictable.
+                                            churn = delta.reparsed.length + delta.vanished.length;
+                                            bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;
+                                            _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                                            if (!bulk) return [
+                                                3,
+                                                18
+                                            ];
+                                            _state.label = 11;
+                                        case 11:
+                                            _state.trys.push([
+                                                11,
+                                                16,
+                                                17,
+                                                18
+                                            ]);
+                                            _iterator1 = _opents.CONTENT_FTS_NAMES[Symbol.iterator]();
+                                            _state.label = 12;
+                                        case 12:
+                                            if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
+                                                3,
+                                                15
+                                            ];
+                                            name = _step1.value;
+                                            return [
+                                                4,
+                                                conn.exec("DROP INDEX IF EXISTS ".concat(name))
+                                            ];
+                                        case 13:
+                                            _state.sent();
+                                            _state.label = 14;
+                                        case 14:
+                                            _iteratorNormalCompletion1 = true;
+                                            return [
+                                                3,
+                                                12
+                                            ];
+                                        case 15:
+                                            return [
+                                                3,
+                                                18
+                                            ];
+                                        case 16:
+                                            err = _state.sent();
+                                            _didIteratorError1 = true;
+                                            _iteratorError1 = err;
+                                            return [
+                                                3,
+                                                18
+                                            ];
+                                        case 17:
+                                            try {
+                                                if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                                                    _iterator1.return();
+                                                }
+                                            } finally{
+                                                if (_didIteratorError1) {
+                                                    throw _iteratorError1;
+                                                }
+                                            }
+                                            return [
+                                                7
+                                            ];
+                                        case 18:
                                             // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5
                                             // content), so vanished and reparsed docs delete in one pass.
                                             contentTouched = _to_consumable_array(vanished).concat(_to_consumable_array(reparsedExisting));
                                             if (!(contentTouched.length > 0)) return [
                                                 3,
-                                                12
+                                                20
                                             ];
                                             return [
                                                 4,
@@ -441,25 +514,86 @@ function reconcile(conn, cfg, baseDir) {
                                                     ];
                                                 }))
                                             ];
-                                        case 11:
+                                        case 19:
                                             _state.sent();
-                                            _state.label = 12;
-                                        case 12:
+                                            _state.label = 20;
+                                        case 20:
                                             if (!(parsedDocs.length > 0)) return [
                                                 3,
-                                                14
+                                                22
                                             ];
                                             return [
                                                 4,
                                                 conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow))
                                             ];
-                                        case 13:
+                                        case 21:
                                             _state.sent();
-                                            _state.label = 14;
-                                        case 14:
+                                            _state.label = 22;
+                                        case 22:
+                                            _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+                                            if (!bulk) return [
+                                                3,
+                                                30
+                                            ];
+                                            _state.label = 23;
+                                        case 23:
+                                            _state.trys.push([
+                                                23,
+                                                28,
+                                                29,
+                                                30
+                                            ]);
+                                            _iterator2 = _opents.CONTENT_FTS_DDL[Symbol.iterator]();
+                                            _state.label = 24;
+                                        case 24:
+                                            if (!!(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done)) return [
+                                                3,
+                                                27
+                                            ];
+                                            ddl = _step2.value;
+                                            return [
+                                                4,
+                                                conn.exec(ddl)
+                                            ];
+                                        case 25:
+                                            _state.sent();
+                                            _state.label = 26;
+                                        case 26:
+                                            _iteratorNormalCompletion2 = true;
+                                            return [
+                                                3,
+                                                24
+                                            ];
+                                        case 27:
+                                            return [
+                                                3,
+                                                30
+                                            ];
+                                        case 28:
+                                            err = _state.sent();
+                                            _didIteratorError2 = true;
+                                            _iteratorError2 = err;
+                                            return [
+                                                3,
+                                                30
+                                            ];
+                                        case 29:
+                                            try {
+                                                if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
+                                                    _iterator2.return();
+                                                }
+                                            } finally{
+                                                if (_didIteratorError2) {
+                                                    throw _iteratorError2;
+                                                }
+                                            }
+                                            return [
+                                                7
+                                            ];
+                                        case 30:
                                             if (!(vanished.length > 0)) return [
                                                 3,
-                                                16
+                                                32
                                             ];
                                             return [
                                                 4,
@@ -469,16 +603,16 @@ function reconcile(conn, cfg, baseDir) {
                                                     ];
                                                 }))
                                             ];
-                                        case 15:
+                                        case 31:
                                             _state.sent();
-                                            _state.label = 16;
-                                        case 16:
+                                            _state.label = 32;
+                                        case 32:
                                             // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
                                             // new docs have nothing to clear, which keeps cold builds linear.
                                             presetTouched = _to_consumable_array(vanished).concat(_to_consumable_array(reparsedExisting));
                                             if (!(presetTouched.length > 0)) return [
                                                 3,
-                                                18
+                                                34
                                             ];
                                             return [
                                                 4,
@@ -488,141 +622,141 @@ function reconcile(conn, cfg, baseDir) {
                                                     ];
                                                 }))
                                             ];
-                                        case 17:
+                                        case 33:
                                             _state.sent();
-                                            _state.label = 18;
-                                        case 18:
+                                            _state.label = 34;
+                                        case 34:
                                             presetRows = [];
-                                            _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined, _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+                                            _iteratorNormalCompletion3 = true, _didIteratorError3 = false, _iteratorError3 = undefined, _iteratorNormalCompletion4 = true, _didIteratorError4 = false, _iteratorError4 = undefined;
                                             try {
-                                                for(_iterator1 = parsedDocs[Symbol.iterator](); !(_iteratorNormalCompletion2 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion2 = true){
-                                                    doc = _step1.value;
+                                                for(_iterator3 = parsedDocs[Symbol.iterator](); !(_iteratorNormalCompletion4 = (_step3 = _iterator3.next()).done); _iteratorNormalCompletion4 = true){
+                                                    doc = _step3.value;
                                                     try {
-                                                        for(_iterator2 = doc.presets[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion1 = true){
-                                                            presetName = _step2.value;
+                                                        for(_iterator4 = doc.presets[Symbol.iterator](); !(_iteratorNormalCompletion3 = (_step4 = _iterator4.next()).done); _iteratorNormalCompletion3 = true){
+                                                            presetName = _step4.value;
                                                             presetRows.push([
                                                                 doc.relPath,
                                                                 presetName
                                                             ]);
                                                         }
                                                     } catch (err) {
-                                                        _didIteratorError1 = true;
-                                                        _iteratorError1 = err;
+                                                        _didIteratorError3 = true;
+                                                        _iteratorError3 = err;
                                                     } finally{
                                                         try {
-                                                            if (!_iteratorNormalCompletion1 && _iterator2.return != null) {
-                                                                _iterator2.return();
+                                                            if (!_iteratorNormalCompletion3 && _iterator4.return != null) {
+                                                                _iterator4.return();
                                                             }
                                                         } finally{
-                                                            if (_didIteratorError1) {
-                                                                throw _iteratorError1;
+                                                            if (_didIteratorError3) {
+                                                                throw _iteratorError3;
                                                             }
                                                         }
                                                     }
                                                 }
                                             } catch (err) {
-                                                _didIteratorError2 = true;
-                                                _iteratorError2 = err;
+                                                _didIteratorError4 = true;
+                                                _iteratorError4 = err;
                                             } finally{
                                                 try {
-                                                    if (!_iteratorNormalCompletion2 && _iterator1.return != null) {
-                                                        _iterator1.return();
+                                                    if (!_iteratorNormalCompletion4 && _iterator3.return != null) {
+                                                        _iterator3.return();
                                                     }
                                                 } finally{
-                                                    if (_didIteratorError2) {
-                                                        throw _iteratorError2;
+                                                    if (_didIteratorError4) {
+                                                        throw _iteratorError4;
                                                     }
                                                 }
                                             }
                                             if (!(presetRows.length > 0)) return [
                                                 3,
-                                                20
+                                                36
                                             ];
                                             return [
                                                 4,
                                                 conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows)
                                             ];
-                                        case 19:
+                                        case 35:
                                             _state.sent();
-                                            _state.label = 20;
-                                        case 20:
+                                            _state.label = 36;
+                                        case 36:
                                             removedPaths = _to_consumable_array(vanished).concat(_to_consumable_array(reparsedExisting));
-                                            _iteratorNormalCompletion3 = true, _didIteratorError3 = false, _iteratorError3 = undefined;
+                                            _iteratorNormalCompletion5 = true, _didIteratorError5 = false, _iteratorError5 = undefined;
                                             if (!(removedPaths.length > 0)) return [
                                                 3,
-                                                28
+                                                44
                                             ];
-                                            _state.label = 21;
-                                        case 21:
+                                            _state.label = 37;
+                                        case 37:
                                             _state.trys.push([
-                                                21,
-                                                26,
-                                                27,
-                                                28
+                                                37,
+                                                42,
+                                                43,
+                                                44
                                             ]);
-                                            _iterator3 = features[Symbol.iterator]();
-                                            _state.label = 22;
-                                        case 22:
-                                            if (!!(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done)) return [
+                                            _iterator5 = features[Symbol.iterator]();
+                                            _state.label = 38;
+                                        case 38:
+                                            if (!!(_iteratorNormalCompletion5 = (_step5 = _iterator5.next()).done)) return [
                                                 3,
-                                                25
+                                                41
                                             ];
-                                            feature = _step3.value;
+                                            feature = _step5.value;
                                             return [
                                                 4,
                                                 (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta)
                                             ];
-                                        case 23:
+                                        case 39:
                                             _state.sent();
-                                            _state.label = 24;
-                                        case 24:
-                                            _iteratorNormalCompletion3 = true;
+                                            _state.label = 40;
+                                        case 40:
+                                            _iteratorNormalCompletion5 = true;
                                             return [
                                                 3,
-                                                22
+                                                38
                                             ];
-                                        case 25:
+                                        case 41:
                                             return [
                                                 3,
-                                                28
+                                                44
                                             ];
-                                        case 26:
+                                        case 42:
                                             err = _state.sent();
-                                            _didIteratorError3 = true;
-                                            _iteratorError3 = err;
+                                            _didIteratorError5 = true;
+                                            _iteratorError5 = err;
                                             return [
                                                 3,
-                                                28
+                                                44
                                             ];
-                                        case 27:
+                                        case 43:
                                             try {
-                                                if (!_iteratorNormalCompletion3 && _iterator3.return != null) {
-                                                    _iterator3.return();
+                                                if (!_iteratorNormalCompletion5 && _iterator5.return != null) {
+                                                    _iterator5.return();
                                                 }
                                             } finally{
-                                                if (_didIteratorError3) {
-                                                    throw _iteratorError3;
+                                                if (_didIteratorError5) {
+                                                    throw _iteratorError5;
                                                 }
                                             }
                                             return [
                                                 7
                                             ];
-                                        case 28:
-                                            _iteratorNormalCompletion4 = true, _didIteratorError4 = false, _iteratorError4 = undefined;
-                                            _state.label = 29;
-                                        case 29:
+                                        case 44:
+                                            _iteratorNormalCompletion6 = true, _didIteratorError6 = false, _iteratorError6 = undefined;
+                                            _state.label = 45;
+                                        case 45:
                                             _state.trys.push([
-                                                29,
-                                                34,
-                                                35,
-                                                36
+                                                45,
+                                                50,
+                                                51,
+                                                52
                                             ]);
                                             _loop = function() {
                                                 var feature, _feature_store, docsForFeature;
                                                 return _ts_generator(this, function(_state) {
                                                     switch(_state.label){
                                                         case 0:
-                                                            feature = _step4.value;
+                                                            feature = _step6.value;
                                                             docsForFeature = parsedDocs.map(function(doc) {
                                                                 return {
                                                                     path: doc.relPath,
@@ -641,117 +775,117 @@ function reconcile(conn, cfg, baseDir) {
                                                     }
                                                 });
                                             };
-                                            _iterator4 = features[Symbol.iterator]();
-                                            _state.label = 30;
-                                        case 30:
-                                            if (!!(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done)) return [
+                                            _iterator6 = features[Symbol.iterator]();
+                                            _state.label = 46;
+                                        case 46:
+                                            if (!!(_iteratorNormalCompletion6 = (_step6 = _iterator6.next()).done)) return [
                                                 3,
-                                                33
+                                                49
                                             ];
                                             return [
                                                 5,
                                                 _ts_values(_loop())
                                             ];
-                                        case 31:
+                                        case 47:
                                             _state.sent();
-                                            _state.label = 32;
-                                        case 32:
-                                            _iteratorNormalCompletion4 = true;
+                                            _state.label = 48;
+                                        case 48:
+                                            _iteratorNormalCompletion6 = true;
                                             return [
                                                 3,
-                                                30
+                                                46
                                             ];
-                                        case 33:
+                                        case 49:
                                             return [
                                                 3,
-                                                36
+                                                52
                                             ];
-                                        case 34:
+                                        case 50:
                                             err = _state.sent();
-                                            _didIteratorError4 = true;
-                                            _iteratorError4 = err;
+                                            _didIteratorError6 = true;
+                                            _iteratorError6 = err;
                                             return [
                                                 3,
-                                                36
+                                                52
                                             ];
-                                        case 35:
+                                        case 51:
                                             try {
-                                                if (!_iteratorNormalCompletion4 && _iterator4.return != null) {
-                                                    _iterator4.return();
+                                                if (!_iteratorNormalCompletion6 && _iterator6.return != null) {
+                                                    _iterator6.return();
                                                 }
                                             } finally{
-                                                if (_didIteratorError4) {
-                                                    throw _iteratorError4;
+                                                if (_didIteratorError6) {
+                                                    throw _iteratorError6;
                                                 }
                                             }
                                             return [
                                                 7
                                             ];
-                                        case 36:
-                                            _iteratorNormalCompletion5 = true, _didIteratorError5 = false, _iteratorError5 = undefined;
-                                            _state.label = 37;
-                                        case 37:
+                                        case 52:
+                                            _iteratorNormalCompletion7 = true, _didIteratorError7 = false, _iteratorError7 = undefined;
+                                            _state.label = 53;
+                                        case 53:
                                             _state.trys.push([
-                                                37,
-                                                42,
-                                                43,
-                                                44
+                                                53,
+                                                58,
+                                                59,
+                                                60
                                             ]);
-                                            _iterator5 = features[Symbol.iterator]();
-                                            _state.label = 38;
-                                        case 38:
-                                            if (!!(_iteratorNormalCompletion5 = (_step5 = _iterator5.next()).done)) return [
+                                            _iterator7 = features[Symbol.iterator]();
+                                            _state.label = 54;
+                                        case 54:
+                                            if (!!(_iteratorNormalCompletion7 = (_step7 = _iterator7.next()).done)) return [
                                                 3,
-                                                41
+                                                57
                                             ];
-                                            feature1 = _step5.value;
+                                            feature1 = _step7.value;
                                             return [
                                                 4,
                                                 (_feature_afterReconcile = feature1.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature1, conn, delta)
                                             ];
-                                        case 39:
+                                        case 55:
                                             _state.sent();
-                                            _state.label = 40;
-                                        case 40:
-                                            _iteratorNormalCompletion5 = true;
+                                            _state.label = 56;
+                                        case 56:
+                                            _iteratorNormalCompletion7 = true;
                                             return [
                                                 3,
-                                                38
+                                                54
                                             ];
-                                        case 41:
+                                        case 57:
                                             return [
                                                 3,
-                                                44
+                                                60
                                             ];
-                                        case 42:
+                                        case 58:
                                             err = _state.sent();
-                                            _didIteratorError5 = true;
-                                            _iteratorError5 = err;
+                                            _didIteratorError7 = true;
+                                            _iteratorError7 = err;
                                             return [
                                                 3,
-                                                44
+                                                60
                                             ];
-                                        case 43:
+                                        case 59:
                                             try {
-                                                if (!_iteratorNormalCompletion5 && _iterator5.return != null) {
-                                                    _iterator5.return();
+                                                if (!_iteratorNormalCompletion7 && _iterator7.return != null) {
+                                                    _iterator7.return();
                                                 }
                                             } finally{
-                                                if (_didIteratorError5) {
-                                                    throw _iteratorError5;
+                                                if (_didIteratorError7) {
+                                                    throw _iteratorError7;
                                                 }
                                             }
                                             return [
                                                 7
                                             ];
-                                        case 44:
+                                        case 60:
                                             return [
                                                 2
                                             ];
                                     }
                                 });
                             })();
-                        })
+                        }, _transactionts.BEGIN_WRITE)
                     ];
                 case 5:
                     _state.sent();