sensemaking 0.19.2 → 0.21.0

package/dist/cjs/store/reconcile.js

@@ -200,7 +200,7 @@ var CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-function reconcile(conn, cfg, baseDir, dialect) {
+function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
     return _async_to_generator(function() {
         var files, currentSet, existingStmt, existingRows, existing, vanished, toReparse, features, seenColumns, report, _ref, parsedDocs, warnings, newColumns, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting, touched, txStart, durationMs;
         return _ts_generator(this, function(_state) {
@@ -228,14 +228,20 @@ function reconcile(conn, cfg, baseDir, dialect) {
                             r
                         ];
                     }));
+                    // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is
+                    // already caught below by !currentSet.has, since it can only be forced by having existed under
+                    // an old preset's match, which means it was reconciled into `existing` already.
                     vanished = existingRows.filter(function(r) {
                         return !currentSet.has(r.path);
                     }).map(function(r) {
                         return r.path;
                     });
+                    // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its
+                    // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.
                     toReparse = files.filter(function(f) {
+                        var _ref;
                         var row = existing.get(f.relPath);
-                        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
+                        return !row || row._mtime !== f.mtimeMs || row._size !== f.size || ((_ref = forcedPaths === null || forcedPaths === void 0 ? void 0 : forcedPaths.has(f.relPath)) !== null && _ref !== void 0 ? _ref : false);
                     });
                     if (vanished.length === 0 && toReparse.length === 0) return [
                         2,
@@ -256,7 +262,9 @@ function reconcile(conn, cfg, baseDir, dialect) {
                     report = (0, _progressts.progress)('reparsing files', toReparse.length);
                     return [
                         4,
-                        (0, _reparsets.reparseFiles)(toReparse, features, cfg, seenColumns, report.tick)
+                        (0, _reparsets.reparseFiles)(toReparse, features, cfg, seenColumns, report.tick, {
+                            pool: pool
+                        })
                     ];
                 case 4:
                     _ref = _state.sent(), parsedDocs = _ref.docs, warnings = _ref.warnings, newColumns = _ref.newColumns;
@@ -326,7 +334,7 @@ function reconcile(conn, cfg, baseDir, dialect) {
                         4,
                         (0, _transactionts.withTransaction)(conn, function() {
                             return _async_to_generator(function() {
-                                var _feature_remove, _feature_afterReconcile, present, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, err, rows, presetRows, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator1, _step1, doc, _iterator2, _step2, presetName, _iteratorNormalCompletion3, _didIteratorError3, _iteratorError3, _iterator3, _step3, feature, err, _iteratorNormalCompletion4, _didIteratorError4, _iteratorError4, _loop, _iterator4, _step4, err, _iteratorNormalCompletion5, _didIteratorError5, _iteratorError5, _iterator5, _step5, feature1, err;
+                                var _feature_remove, _feature_afterReconcile, present, missingColumns, contentTouched, currentPathsStmt, currentPaths, _, staleAdded, rows, presetRows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator, _step, doc, _iterator1, _step1, presetName, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, feature, err, _iteratorNormalCompletion3, _didIteratorError3, _iteratorError3, _loop, _iterator3, _step3, err, _iteratorNormalCompletion4, _didIteratorError4, _iteratorError4, _iterator4, _step4, feature1, err;
                                 return _ts_generator(this, function(_state) {
                                     switch(_state.label){
                                         case 0:
@@ -336,70 +344,58 @@ function reconcile(conn, cfg, baseDir, dialect) {
                                             ];
                                         case 1:
                                             present = _state.sent();
-                                            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                                            _state.label = 2;
+                                            missingColumns = newColumns.filter(function(col) {
+                                                return !present.has(col);
+                                            });
+                                            if (!(missingColumns.length > 0)) return [
+                                                3,
+                                                3
+                                            ];
+                                            return [
+                                                4,
+                                                dialect.addColumns(conn, missingColumns)
+                                            ];
                                         case 2:
-                                            _state.trys.push([
-                                                2,
-                                                7,
-                                                8,
-                                                9
-                                            ]);
-                                            _iterator = newColumns[Symbol.iterator]();
+                                            _state.sent();
                                             _state.label = 3;
                                         case 3:
-                                            if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                                            // Revalidated once the lock is held, before this process's own frontmatter upsert below:
+                                            // `added` came from a path read taken before this transaction's lock, so a path still
+                                            // called "added" here may already have a content row from a concurrent reconcile that
+                                            // committed while this one waited. One SELECT, not a DELETE per added file -- with no
+                                            // contention it returns the same set `existing` already ruled out, so nothing extra clears.
+                                            contentTouched = touched;
+                                            if (!(added.length > 0)) return [
                                                 3,
                                                 6
                                             ];
-                                            col = _step.value;
-                                            if (!!present.has(col)) return [
-                                                3,
-                                                5
-                                            ];
                                             return [
                                                 4,
-                                                conn.exec("ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(col)).concat(dialect.columnTypeSuffix()))
+                                                conn.prepare('SELECT "path" FROM frontmatter')
                                             ];
                                         case 4:
-                                            _state.sent();
-                                            _state.label = 5;
-                                        case 5:
-                                            _iteratorNormalCompletion = true;
+                                            currentPathsStmt = _state.sent();
+                                            _ = Set.bind;
                                             return [
-                                                3,
-                                                3
+                                                4,
+                                                currentPathsStmt.all()
                                             ];
+                                        case 5:
+                                            currentPaths = new (_.apply(Set, [
+                                                void 0,
+                                                _state.sent().map(function(r) {
+                                                    return r.path;
+                                                })
+                                            ]));
+                                            staleAdded = added.filter(function(p) {
+                                                return currentPaths.has(p);
+                                            });
+                                            if (staleAdded.length > 0) contentTouched = _to_consumable_array(touched).concat(_to_consumable_array(staleAdded));
+                                            _state.label = 6;
                                         case 6:
-                                            return [
-                                                3,
-                                                9
-                                            ];
-                                        case 7:
-                                            err = _state.sent();
-                                            _didIteratorError = true;
-                                            _iteratorError = err;
-                                            return [
-                                                3,
-                                                9
-                                            ];
-                                        case 8:
-                                            try {
-                                                if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                                    _iterator.return();
-                                                }
-                                            } finally{
-                                                if (_didIteratorError) {
-                                                    throw _iteratorError;
-                                                }
-                                            }
-                                            return [
-                                                7
-                                            ];
-                                        case 9:
                                             if (!(parsedDocs.length > 0)) return [
                                                 3,
-                                                11
+                                                8
                                             ];
                                             rows = parsedDocs.map(function(doc) {
                                                 return writableColumns.map(function(col) {
@@ -417,21 +413,21 @@ function reconcile(conn, cfg, baseDir, dialect) {
                                                 4,
                                                 conn.runBatch(insertSql, rows)
                                             ];
-                                        case 10:
+                                        case 7:
                                             _state.sent();
-                                            _state.label = 11;
-                                        case 11:
+                                            _state.label = 8;
+                                        case 8:
                                             // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
                                             // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.
                                             return [
                                                 4,
-                                                dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg)
+                                                dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg)
                                             ];
-                                        case 12:
+                                        case 9:
                                             _state.sent();
                                             if (!(touched.length > 0)) return [
                                                 3,
-                                                14
+                                                11
                                             ];
                                             return [
                                                 4,
@@ -441,67 +437,67 @@ function reconcile(conn, cfg, baseDir, dialect) {
                                                     ];
                                                 }))
                                             ];
-                                        case 13:
+                                        case 10:
                                             _state.sent();
-                                            _state.label = 14;
-                                        case 14:
+                                            _state.label = 11;
+                                        case 11:
                                             presetRows = [];
-                                            _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined, _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+                                            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined, _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                                             try {
-                                                for(_iterator1 = parsedDocs[Symbol.iterator](); !(_iteratorNormalCompletion2 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion2 = true){
-                                                    doc = _step1.value;
+                                                for(_iterator = parsedDocs[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step = _iterator.next()).done); _iteratorNormalCompletion1 = true){
+                                                    doc = _step.value;
                                                     try {
-                                                        for(_iterator2 = doc.presets[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion1 = true){
-                                                            presetName = _step2.value;
+                                                        for(_iterator1 = doc.presets[Symbol.iterator](); !(_iteratorNormalCompletion = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion = true){
+                                                            presetName = _step1.value;
                                                             presetRows.push([
                                                                 doc.relPath,
                                                                 presetName
                                                             ]);
                                                         }
                                                     } catch (err) {
-                                                        _didIteratorError1 = true;
-                                                        _iteratorError1 = err;
+                                                        _didIteratorError = true;
+                                                        _iteratorError = err;
                                                     } finally{
                                                         try {
-                                                            if (!_iteratorNormalCompletion1 && _iterator2.return != null) {
-                                                                _iterator2.return();
+                                                            if (!_iteratorNormalCompletion && _iterator1.return != null) {
+                                                                _iterator1.return();
                                                             }
                                                         } finally{
-                                                            if (_didIteratorError1) {
-                                                                throw _iteratorError1;
+                                                            if (_didIteratorError) {
+                                                                throw _iteratorError;
                                                             }
                                                         }
                                                     }
                                                 }
                                             } catch (err) {
-                                                _didIteratorError2 = true;
-                                                _iteratorError2 = err;
+                                                _didIteratorError1 = true;
+                                                _iteratorError1 = err;
                                             } finally{
                                                 try {
-                                                    if (!_iteratorNormalCompletion2 && _iterator1.return != null) {
-                                                        _iterator1.return();
+                                                    if (!_iteratorNormalCompletion1 && _iterator.return != null) {
+                                                        _iterator.return();
                                                     }
                                                 } finally{
-                                                    if (_didIteratorError2) {
-                                                        throw _iteratorError2;
+                                                    if (_didIteratorError1) {
+                                                        throw _iteratorError1;
                                                     }
                                                 }
                                             }
                                             if (!(presetRows.length > 0)) return [
                                                 3,
-                                                16
+                                                13
                                             ];
                                             return [
                                                 4,
-                                                conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows)
+                                                conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows)
                                             ];
-                                        case 15:
+                                        case 12:
                                             _state.sent();
-                                            _state.label = 16;
-                                        case 16:
+                                            _state.label = 13;
+                                        case 13:
                                             if (!(vanished.length > 0)) return [
                                                 3,
-                                                18
+                                                15
                                             ];
                                             return [
                                                 4,
@@ -511,86 +507,86 @@ function reconcile(conn, cfg, baseDir, dialect) {
                                                     ];
                                                 }))
                                             ];
-                                        case 17:
+                                        case 14:
                                             _state.sent();
-                                            _state.label = 18;
-                                        case 18:
-                                            _iteratorNormalCompletion3 = true, _didIteratorError3 = false, _iteratorError3 = undefined;
+                                            _state.label = 15;
+                                        case 15:
+                                            _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
                                             if (!(touched.length > 0)) return [
                                                 3,
-                                                26
+                                                23
                                             ];
-                                            _state.label = 19;
-                                        case 19:
+                                            _state.label = 16;
+                                        case 16:
                                             _state.trys.push([
-                                                19,
-                                                24,
-                                                25,
-                                                26
+                                                16,
+                                                21,
+                                                22,
+                                                23
                                             ]);
-                                            _iterator3 = features[Symbol.iterator]();
-                                            _state.label = 20;
-                                        case 20:
-                                            if (!!(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done)) return [
+                                            _iterator2 = features[Symbol.iterator]();
+                                            _state.label = 17;
+                                        case 17:
+                                            if (!!(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done)) return [
                                                 3,
-                                                23
+                                                20
                                             ];
-                                            feature = _step3.value;
+                                            feature = _step2.value;
                                             return [
                                                 4,
                                                 (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta)
                                             ];
-                                        case 21:
+                                        case 18:
                                             _state.sent();
-                                            _state.label = 22;
-                                        case 22:
-                                            _iteratorNormalCompletion3 = true;
+                                            _state.label = 19;
+                                        case 19:
+                                            _iteratorNormalCompletion2 = true;
                                             return [
                                                 3,
-                                                20
+                                                17
                                             ];
-                                        case 23:
+                                        case 20:
                                             return [
                                                 3,
-                                                26
+                                                23
                                             ];
-                                        case 24:
+                                        case 21:
                                             err = _state.sent();
-                                            _didIteratorError3 = true;
-                                            _iteratorError3 = err;
+                                            _didIteratorError2 = true;
+                                            _iteratorError2 = err;
                                             return [
                                                 3,
-                                                26
+                                                23
                                             ];
-                                        case 25:
+                                        case 22:
                                             try {
-                                                if (!_iteratorNormalCompletion3 && _iterator3.return != null) {
-                                                    _iterator3.return();
+                                                if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
+                                                    _iterator2.return();
                                                 }
                                             } finally{
-                                                if (_didIteratorError3) {
-                                                    throw _iteratorError3;
+                                                if (_didIteratorError2) {
+                                                    throw _iteratorError2;
                                                 }
                                             }
                                             return [
                                                 7
                                             ];
-                                        case 26:
-                                            _iteratorNormalCompletion4 = true, _didIteratorError4 = false, _iteratorError4 = undefined;
-                                            _state.label = 27;
-                                        case 27:
+                                        case 23:
+                                            _iteratorNormalCompletion3 = true, _didIteratorError3 = false, _iteratorError3 = undefined;
+                                            _state.label = 24;
+                                        case 24:
                                             _state.trys.push([
-                                                27,
-                                                32,
-                                                33,
-                                                34
+                                                24,
+                                                29,
+                                                30,
+                                                31
                                             ]);
                                             _loop = function() {
                                                 var feature, _feature_store, docsForFeature;
                                                 return _ts_generator(this, function(_state) {
                                                     switch(_state.label){
                                                         case 0:
-                                                            feature = _step4.value;
+                                                            feature = _step3.value;
                                                             docsForFeature = parsedDocs.map(function(doc) {
                                                                 return {
                                                                     path: doc.relPath,
@@ -609,110 +605,110 @@ function reconcile(conn, cfg, baseDir, dialect) {
                                                     }
                                                 });
                                             };
-                                            _iterator4 = features[Symbol.iterator]();
-                                            _state.label = 28;
-                                        case 28:
-                                            if (!!(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done)) return [
+                                            _iterator3 = features[Symbol.iterator]();
+                                            _state.label = 25;
+                                        case 25:
+                                            if (!!(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done)) return [
                                                 3,
-                                                31
+                                                28
                                             ];
                                             return [
                                                 5,
                                                 _ts_values(_loop())
                                             ];
-                                        case 29:
+                                        case 26:
                                             _state.sent();
-                                            _state.label = 30;
-                                        case 30:
-                                            _iteratorNormalCompletion4 = true;
+                                            _state.label = 27;
+                                        case 27:
+                                            _iteratorNormalCompletion3 = true;
                                             return [
                                                 3,
-                                                28
+                                                25
                                             ];
-                                        case 31:
+                                        case 28:
                                             return [
                                                 3,
-                                                34
+                                                31
                                             ];
-                                        case 32:
+                                        case 29:
                                             err = _state.sent();
-                                            _didIteratorError4 = true;
-                                            _iteratorError4 = err;
+                                            _didIteratorError3 = true;
+                                            _iteratorError3 = err;
                                             return [
                                                 3,
-                                                34
+                                                31
                                             ];
-                                        case 33:
+                                        case 30:
                                             try {
-                                                if (!_iteratorNormalCompletion4 && _iterator4.return != null) {
-                                                    _iterator4.return();
+                                                if (!_iteratorNormalCompletion3 && _iterator3.return != null) {
+                                                    _iterator3.return();
                                                 }
                                             } finally{
-                                                if (_didIteratorError4) {
-                                                    throw _iteratorError4;
+                                                if (_didIteratorError3) {
+                                                    throw _iteratorError3;
                                                 }
                                             }
                                             return [
                                                 7
                                             ];
-                                        case 34:
-                                            _iteratorNormalCompletion5 = true, _didIteratorError5 = false, _iteratorError5 = undefined;
-                                            _state.label = 35;
-                                        case 35:
+                                        case 31:
+                                            _iteratorNormalCompletion4 = true, _didIteratorError4 = false, _iteratorError4 = undefined;
+                                            _state.label = 32;
+                                        case 32:
                                             _state.trys.push([
-                                                35,
-                                                40,
-                                                41,
-                                                42
+                                                32,
+                                                37,
+                                                38,
+                                                39
                                             ]);
-                                            _iterator5 = features[Symbol.iterator]();
-                                            _state.label = 36;
-                                        case 36:
-                                            if (!!(_iteratorNormalCompletion5 = (_step5 = _iterator5.next()).done)) return [
+                                            _iterator4 = features[Symbol.iterator]();
+                                            _state.label = 33;
+                                        case 33:
+                                            if (!!(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done)) return [
                                                 3,
-                                                39
+                                                36
                                             ];
-                                            feature1 = _step5.value;
+                                            feature1 = _step4.value;
                                             return [
                                                 4,
                                                 (_feature_afterReconcile = feature1.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature1, conn, delta)
                                             ];
-                                        case 37:
+                                        case 34:
                                             _state.sent();
-                                            _state.label = 38;
-                                        case 38:
-                                            _iteratorNormalCompletion5 = true;
+                                            _state.label = 35;
+                                        case 35:
+                                            _iteratorNormalCompletion4 = true;
                                             return [
                                                 3,
-                                                36
+                                                33
                                             ];
-                                        case 39:
+                                        case 36:
                                             return [
                                                 3,
-                                                42
+                                                39
                                             ];
-                                        case 40:
+                                        case 37:
                                             err = _state.sent();
-                                            _didIteratorError5 = true;
-                                            _iteratorError5 = err;
+                                            _didIteratorError4 = true;
+                                            _iteratorError4 = err;
                                             return [
                                                 3,
-                                                42
+                                                39
                                             ];
-                                        case 41:
+                                        case 38:
                                             try {
-                                                if (!_iteratorNormalCompletion5 && _iterator5.return != null) {
-                                                    _iterator5.return();
+                                                if (!_iteratorNormalCompletion4 && _iterator4.return != null) {
+                                                    _iterator4.return();
                                                 }
                                             } finally{
-                                                if (_didIteratorError5) {
-                                                    throw _iteratorError5;
+                                                if (_didIteratorError4) {
+                                                    throw _iteratorError4;
                                                 }
                                             }
                                             return [
                                                 7
                                             ];
-                                        case 42:
+                                        case 39:
                                             return [
                                                 2
                                             ];