sensemaking 0.25.0 → 0.26.0

package/dist/cjs/store/reconcile.js

@@ -12,6 +12,15 @@ _export(exports, {
     get CORE_FRONTMATTER_COLUMNS () {
         return CORE_FRONTMATTER_COLUMNS;
     },
+    get createReconcileTiming () {
+        return createReconcileTiming;
+    },
+    get prepareReconcile () {
+        return prepareReconcile;
+    },
+    get readReconcileSnapshot () {
+        return readReconcileSnapshot;
+    },
     get reconcile () {
         return reconcile;
     }
@@ -203,35 +212,28 @@ var CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
+function createReconcileTiming(cfg) {
+    var start = process.hrtime.bigint();
+    return {
+        stages: (0, _stagests.stageRecorder)((0, _indexts.activeFeatures)(cfg).map(function(feature) {
+            return feature.name;
+        })),
+        elapsed: function elapsed() {
+            return Number(process.hrtime.bigint() - start) / 1e6;
+        },
+        workerParseMs: 0,
+        txMs: 0
+    };
+}
+function readReconcileSnapshot(conn, timing) {
     return _async_to_generator(function() {
-        var rootDir, configDir, start, features, stages, elapsed, files, currentSet, existingRows, existing, vanished, toReparse, seenColumns, report, _ref, parsedDocs, warnings, newColumns, workerParseMs, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting, touched, txStart, durationMs;
+        var existingRows, _tmp;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    rootDir = paths.rootDir, configDir = paths.configDir;
-                    start = process.hrtime.bigint();
-                    features = (0, _indexts.activeFeatures)(cfg);
-                    stages = (0, _stagests.stageRecorder)(features.map(function(f) {
-                        return f.name;
-                    }));
-                    elapsed = function elapsed() {
-                        return Number(process.hrtime.bigint() - start) / 1e6;
-                    };
                     return [
                         4,
-                        stages.time('list', function() {
-                            return (0, _indexts1.listFiles)(cfg, rootDir);
-                        })
-                    ];
-                case 1:
-                    files = _state.sent();
-                    currentSet = new Set(files.map(function(f) {
-                        return f.relPath;
-                    }));
-                    return [
-                        4,
-                        stages.time('existing', function() {
+                        timing.stages.time('existing', function() {
                             return _async_to_generator(function() {
                                 var existingStmt;
                                 return _ts_generator(this, function(_state) {
@@ -257,8 +259,45 @@ function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
                             })();
                         })
                     ];
-                case 2:
+                case 1:
                     existingRows = _state.sent();
+                    _tmp = {
+                        existingRows: existingRows
+                    };
+                    return [
+                        4,
+                        (0, _sharedts.getColumns)(conn)
+                    ];
+                case 2:
+                    return [
+                        2,
+                        (_tmp.seenColumns = _state.sent(), _tmp)
+                    ];
+            }
+        });
+    })();
+}
+function prepareReconcile(snapshot, cfg, paths, dialect, timing, pool, forcedPaths) {
+    return _async_to_generator(function() {
+        var rootDir, configDir, features, stages, elapsed, files, currentSet, existingRows, existing, vanished, toReparse, seenColumns, report, _ref, parsedDocs, warnings, newColumns, workerParseMs, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting, touched;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    rootDir = paths.rootDir, configDir = paths.configDir;
+                    features = (0, _indexts.activeFeatures)(cfg);
+                    stages = timing.stages, elapsed = timing.elapsed;
+                    return [
+                        4,
+                        stages.time('list', function() {
+                            return (0, _indexts1.listFiles)(cfg, rootDir);
+                        })
+                    ];
+                case 1:
+                    files = _state.sent();
+                    currentSet = new Set(files.map(function(f) {
+                        return f.relPath;
+                    }));
+                    existingRows = snapshot.existingRows;
                     existing = new Map(existingRows.map(function(r) {
                         return [
                             r.path,
@@ -280,20 +319,38 @@ function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
                         var row = existing.get(f.relPath);
                         return !row || row._mtime !== f.mtimeMs || row._size !== f.size || ((_ref = forcedPaths === null || forcedPaths === void 0 ? void 0 : forcedPaths.has(f.relPath)) !== null && _ref !== void 0 ? _ref : false);
                     });
-                    if (vanished.length === 0 && toReparse.length === 0) return [
-                        2,
-                        {
-                            parsed: 0,
-                            warnings: [],
-                            stages: stages.take(elapsed(), 0)
-                        }
-                    ];
-                    return [
-                        4,
-                        (0, _sharedts.getColumns)(conn)
-                    ];
-                case 3:
-                    seenColumns = _state.sent();
+                    if (vanished.length === 0 && toReparse.length === 0) {
+                        return [
+                            2,
+                            {
+                                mutated: false,
+                                parsed: 0,
+                                warnings: [],
+                                apply: function apply() {
+                                    return _async_to_generator(function() {
+                                        return _ts_generator(this, function(_state) {
+                                            return [
+                                                2
+                                            ];
+                                        });
+                                    })();
+                                },
+                                finish: function finish(txMs) {
+                                    return stages.take(elapsed(), txMs, timing.workerParseMs);
+                                },
+                                record: function record() {
+                                    return _async_to_generator(function() {
+                                        return _ts_generator(this, function(_state) {
+                                            return [
+                                                2
+                                            ];
+                                        });
+                                    })();
+                                }
+                            }
+                        ];
+                    }
+                    seenColumns = new Set(snapshot.seenColumns);
                     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
                     // reconciles stay silent (progress() has a threshold).
                     report = (0, _progressts.progress)('reparsing files', toReparse.length);
@@ -305,8 +362,9 @@ function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
                             });
                         })
                     ];
-                case 4:
+                case 2:
                     _ref = _state.sent(), parsedDocs = _ref.docs, warnings = _ref.warnings, newColumns = _ref.newColumns, workerParseMs = _ref.workerParseMs;
+                    timing.workerParseMs += workerParseMs;
                     report.finish();
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
@@ -368,575 +426,611 @@ function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
                     // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to
                     // be reinserted fresh. Disjoint from `added`, which has nothing to clear.
                     touched = _to_consumable_array(vanished).concat(_to_consumable_array(reparsedExisting));
-                    txStart = Date.now();
                     return [
-                        4,
-                        (0, _transactionts.withTransaction)(conn, function() {
-                            return _async_to_generator(function() {
-                                var present, missingColumns, contentTouched, toRow, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step, err, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _loop1, _iterator1, _step1, err, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _loop2, _iterator2, _step2, err;
-                                return _ts_generator(this, function(_state) {
-                                    switch(_state.label){
-                                        case 0:
-                                            return [
-                                                4,
-                                                (0, _sharedts.getColumns)(conn)
-                                            ];
-                                        case 1:
-                                            present = _state.sent();
-                                            missingColumns = newColumns.filter(function(col) {
-                                                return !present.has(col);
-                                            });
-                                            if (!(missingColumns.length > 0)) return [
-                                                3,
-                                                3
-                                            ];
-                                            return [
-                                                4,
-                                                stages.time('alter', function() {
-                                                    return dialect.addColumns(conn, missingColumns);
-                                                })
-                                            ];
-                                        case 2:
-                                            _state.sent();
-                                            _state.label = 3;
-                                        case 3:
-                                            // Revalidated once the lock is held, before this process's own frontmatter upsert below:
-                                            // `added` came from a path read taken before this transaction's lock, so a path still
-                                            // called "added" here may already have a content row from a concurrent reconcile that
-                                            // committed while this one waited. One SELECT, not a DELETE per added file -- with no
-                                            // contention it returns the same set `existing` already ruled out, so nothing extra clears.
-                                            contentTouched = touched;
-                                            if (!(added.length > 0)) return [
-                                                3,
-                                                5
-                                            ];
-                                            return [
-                                                4,
-                                                stages.time('added-recheck', function() {
-                                                    return _async_to_generator(function() {
-                                                        var currentPathsStmt, currentPaths, _, staleAdded;
-                                                        return _ts_generator(this, function(_state) {
-                                                            switch(_state.label){
-                                                                case 0:
-                                                                    return [
-                                                                        4,
-                                                                        conn.prepare('SELECT "path" FROM frontmatter')
-                                                                    ];
-                                                                case 1:
-                                                                    currentPathsStmt = _state.sent();
-                                                                    _ = Set.bind;
-                                                                    return [
-                                                                        4,
-                                                                        currentPathsStmt.all()
-                                                                    ];
-                                                                case 2:
-                                                                    currentPaths = new (_.apply(Set, [
-                                                                        void 0,
-                                                                        _state.sent().map(function(r) {
-                                                                            return r.path;
-                                                                        })
-                                                                    ]));
-                                                                    staleAdded = added.filter(function(p) {
-                                                                        return currentPaths.has(p);
-                                                                    });
-                                                                    return [
-                                                                        2,
-                                                                        staleAdded.length > 0 ? _to_consumable_array(touched).concat(_to_consumable_array(staleAdded)) : touched
-                                                                    ];
-                                                            }
-                                                        });
-                                                    })();
-                                                })
-                                            ];
-                                        case 4:
-                                            contentTouched = _state.sent();
-                                            _state.label = 5;
-                                        case 5:
-                                            if (!(parsedDocs.length > 0)) return [
-                                                3,
-                                                7
-                                            ];
-                                            toRow = function toRow(doc) {
-                                                return writableColumns.map(function(col) {
-                                                    var _doc_data_col;
-                                                    if (col === 'path') return doc.relPath;
-                                                    if (col === '_mtime') return doc.mtimeMs;
-                                                    if (col === '_ctime') return doc.ctimeMs;
-                                                    if (col === '_size') return doc.size;
-                                                    // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
-                                                    if (col === '_parse_error') return doc.parseError;
-                                                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+                        2,
+                        {
+                            mutated: true,
+                            parsed: parsedDocs.length,
+                            warnings: warnings,
+                            apply: function apply(conn) {
+                                return _async_to_generator(function() {
+                                    var present, missingColumns, toRow, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step, err, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _loop1, _iterator1, _step1, err, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _loop2, _iterator2, _step2, err;
+                                    return _ts_generator(this, function(_state) {
+                                        switch(_state.label){
+                                            case 0:
+                                                return [
+                                                    4,
+                                                    (0, _sharedts.getColumns)(conn)
+                                                ];
+                                            case 1:
+                                                present = _state.sent();
+                                                missingColumns = newColumns.filter(function(col) {
+                                                    return !present.has(col);
                                                 });
-                                            };
-                                            // A path in `added` has no existing frontmatter row, so it can never conflict; the rest
-                                            // genuinely can, and keep the upsert.
-                                            return [
-                                                4,
-                                                stages.time('fm-upsert', function() {
-                                                    return _async_to_generator(function() {
-                                                        var newDocs, updateDocs;
-                                                        return _ts_generator(this, function(_state) {
-                                                            switch(_state.label){
-                                                                case 0:
-                                                                    newDocs = parsedDocs.filter(function(d) {
-                                                                        return addedSet.has(d.relPath);
-                                                                    });
-                                                                    updateDocs = parsedDocs.filter(function(d) {
-                                                                        return !addedSet.has(d.relPath);
-                                                                    });
-                                                                    return [
-                                                                        4,
-                                                                        (0, _sharedts.appendRows)(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow))
-                                                                    ];
-                                                                case 1:
-                                                                    _state.sent();
-                                                                    if (!(updateDocs.length > 0)) return [
-                                                                        3,
-                                                                        3
-                                                                    ];
-                                                                    return [
-                                                                        4,
-                                                                        conn.runBatch(insertSql, updateDocs.map(toRow))
-                                                                    ];
-                                                                case 2:
-                                                                    _state.sent();
-                                                                    _state.label = 3;
-                                                                case 3:
-                                                                    return [
-                                                                        2
-                                                                    ];
-                                                            }
-                                                        });
-                                                    })();
-                                                })
-                                            ];
-                                        case 6:
-                                            _state.sent();
-                                            _state.label = 7;
-                                        case 7:
-                                            // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
-                                            // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.
-                                            return [
-                                                4,
-                                                stages.time('text-index', function() {
-                                                    return dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg);
-                                                })
-                                            ];
-                                        case 8:
-                                            _state.sent();
-                                            if (!(contentTouched.length > 0)) return [
-                                                3,
-                                                10
-                                            ];
-                                            return [
-                                                4,
-                                                conn.runBatch('DELETE FROM indexed_sources WHERE "path" = ?', contentTouched.map(function(p) {
-                                                    return [
-                                                        p
-                                                    ];
-                                                }))
-                                            ];
-                                        case 9:
-                                            _state.sent();
-                                            _state.label = 10;
-                                        case 10:
-                                            return [
-                                                4,
-                                                (0, _sharedts.appendRows)(conn, 'indexed_sources', [
-                                                    'path',
-                                                    'text'
-                                                ], 'INSERT INTO indexed_sources ("path", text) VALUES (?, ?) ON CONFLICT("path") DO UPDATE SET text = excluded.text', parsedDocs.map(function(doc) {
-                                                    return [
-                                                        doc.relPath,
-                                                        doc.source
-                                                    ];
-                                                }))
-                                            ];
-                                        case 11:
-                                            _state.sent();
-                                            // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
-                                            // new docs have nothing to clear, which keeps cold builds linear.
-                                            return [
-                                                4,
-                                                stages.time('presets', function() {
-                                                    return _async_to_generator(function() {
-                                                        var presetRows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator, _step, doc, _iterator1, _step1, presetName;
-                                                        return _ts_generator(this, function(_state) {
-                                                            switch(_state.label){
-                                                                case 0:
-                                                                    if (!(touched.length > 0)) return [
-                                                                        3,
-                                                                        2
-                                                                    ];
-                                                                    return [
-                                                                        4,
-                                                                        conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map(function(p) {
-                                                                            return [
-                                                                                p
-                                                                            ];
-                                                                        }))
-                                                                    ];
-                                                                case 1:
-                                                                    _state.sent();
-                                                                    _state.label = 2;
-                                                                case 2:
-                                                                    presetRows = [];
-                                                                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined, _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
-                                                                    try {
-                                                                        for(_iterator = parsedDocs[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step = _iterator.next()).done); _iteratorNormalCompletion1 = true){
-                                                                            doc = _step.value;
-                                                                            try {
-                                                                                for(_iterator1 = doc.presets[Symbol.iterator](); !(_iteratorNormalCompletion = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion = true){
-                                                                                    presetName = _step1.value;
-                                                                                    presetRows.push([
-                                                                                        doc.relPath,
-                                                                                        presetName
-                                                                                    ]);
-                                                                                }
-                                                                            } catch (err) {
-                                                                                _didIteratorError = true;
-                                                                                _iteratorError = err;
-                                                                            } finally{
+                                                if (!(missingColumns.length > 0)) return [
+                                                    3,
+                                                    3
+                                                ];
+                                                return [
+                                                    4,
+                                                    stages.time('alter', function() {
+                                                        return dialect.addColumns(conn, missingColumns);
+                                                    })
+                                                ];
+                                            case 2:
+                                                _state.sent();
+                                                _state.label = 3;
+                                            case 3:
+                                                if (!(parsedDocs.length > 0)) return [
+                                                    3,
+                                                    5
+                                                ];
+                                                toRow = function toRow(doc) {
+                                                    return writableColumns.map(function(col) {
+                                                        var _doc_data_col;
+                                                        if (col === 'path') return doc.relPath;
+                                                        if (col === '_mtime') return doc.mtimeMs;
+                                                        if (col === '_ctime') return doc.ctimeMs;
+                                                        if (col === '_size') return doc.size;
+                                                        if (col === '_parse_error') return doc.parseError;
+                                                        return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+                                                    });
+                                                };
+                                                return [
+                                                    4,
+                                                    stages.time('fm-upsert', function() {
+                                                        return _async_to_generator(function() {
+                                                            var newDocs, updateDocs, rows;
+                                                            return _ts_generator(this, function(_state) {
+                                                                switch(_state.label){
+                                                                    case 0:
+                                                                        newDocs = parsedDocs.filter(function(d) {
+                                                                            return addedSet.has(d.relPath);
+                                                                        });
+                                                                        updateDocs = parsedDocs.filter(function(d) {
+                                                                            return !addedSet.has(d.relPath);
+                                                                        });
+                                                                        return [
+                                                                            4,
+                                                                            (0, _sharedts.appendRows)(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow))
+                                                                        ];
+                                                                    case 1:
+                                                                        _state.sent();
+                                                                        if (!(updateDocs.length > 0)) return [
+                                                                            3,
+                                                                            5
+                                                                        ];
+                                                                        rows = updateDocs.map(toRow);
+                                                                        if (!dialect.updateFrontmatter) return [
+                                                                            3,
+                                                                            3
+                                                                        ];
+                                                                        return [
+                                                                            4,
+                                                                            dialect.updateFrontmatter(conn, writableColumns, rows)
+                                                                        ];
+                                                                    case 2:
+                                                                        _state.sent();
+                                                                        return [
+                                                                            3,
+                                                                            5
+                                                                        ];
+                                                                    case 3:
+                                                                        return [
+                                                                            4,
+                                                                            conn.runBatch(insertSql, rows)
+                                                                        ];
+                                                                    case 4:
+                                                                        _state.sent();
+                                                                        _state.label = 5;
+                                                                    case 5:
+                                                                        return [
+                                                                            2
+                                                                        ];
+                                                                }
+                                                            });
+                                                        })();
+                                                    })
+                                                ];
+                                            case 4:
+                                                _state.sent();
+                                                _state.label = 5;
+                                            case 5:
+                                                return [
+                                                    4,
+                                                    stages.time('text-index', function() {
+                                                        return dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);
+                                                    })
+                                                ];
+                                            case 6:
+                                                _state.sent();
+                                                if (!(touched.length > 0)) return [
+                                                    3,
+                                                    8
+                                                ];
+                                                return [
+                                                    4,
+                                                    conn.runBatch('DELETE FROM indexed_sources WHERE "path" = ?', touched.map(function(p) {
+                                                        return [
+                                                            p
+                                                        ];
+                                                    }))
+                                                ];
+                                            case 7:
+                                                _state.sent();
+                                                _state.label = 8;
+                                            case 8:
+                                                return [
+                                                    4,
+                                                    (0, _sharedts.appendRows)(conn, 'indexed_sources', [
+                                                        'path',
+                                                        'text'
+                                                    ], 'INSERT INTO indexed_sources ("path", text) VALUES (?, ?) ON CONFLICT("path") DO UPDATE SET text = excluded.text', parsedDocs.map(function(doc) {
+                                                        return [
+                                                            doc.relPath,
+                                                            doc.source
+                                                        ];
+                                                    }))
+                                                ];
+                                            case 9:
+                                                _state.sent();
+                                                return [
+                                                    4,
+                                                    stages.time('presets', function() {
+                                                        return _async_to_generator(function() {
+                                                            var presetRows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator, _step, doc, _iterator1, _step1, presetName;
+                                                            return _ts_generator(this, function(_state) {
+                                                                switch(_state.label){
+                                                                    case 0:
+                                                                        if (!(touched.length > 0)) return [
+                                                                            3,
+                                                                            2
+                                                                        ];
+                                                                        return [
+                                                                            4,
+                                                                            conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map(function(p) {
+                                                                                return [
+                                                                                    p
+                                                                                ];
+                                                                            }))
+                                                                        ];
+                                                                    case 1:
+                                                                        _state.sent();
+                                                                        _state.label = 2;
+                                                                    case 2:
+                                                                        presetRows = [];
+                                                                        _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined, _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                                                                        try {
+                                                                            for(_iterator = parsedDocs[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step = _iterator.next()).done); _iteratorNormalCompletion1 = true){
+                                                                                doc = _step.value;
                                                                                 try {
-                                                                                    if (!_iteratorNormalCompletion && _iterator1.return != null) {
-                                                                                        _iterator1.return();
+                                                                                    for(_iterator1 = doc.presets[Symbol.iterator](); !(_iteratorNormalCompletion = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion = true){
+                                                                                        presetName = _step1.value;
+                                                                                        presetRows.push([
+                                                                                            doc.relPath,
+                                                                                            presetName
+                                                                                        ]);
                                                                                     }
+                                                                                } catch (err) {
+                                                                                    _didIteratorError = true;
+                                                                                    _iteratorError = err;
                                                                                 } finally{
-                                                                                    if (_didIteratorError) {
-                                                                                        throw _iteratorError;
+                                                                                    try {
+                                                                                        if (!_iteratorNormalCompletion && _iterator1.return != null) {
+                                                                                            _iterator1.return();
+                                                                                        }
+                                                                                    } finally{
+                                                                                        if (_didIteratorError) {
+                                                                                            throw _iteratorError;
+                                                                                        }
                                                                                     }
                                                                                 }
                                                                             }
-                                                                        }
-                                                                    } catch (err) {
-                                                                        _didIteratorError1 = true;
-                                                                        _iteratorError1 = err;
-                                                                    } finally{
-                                                                        try {
-                                                                            if (!_iteratorNormalCompletion1 && _iterator.return != null) {
-                                                                                _iterator.return();
-                                                                            }
+                                                                        } catch (err) {
+                                                                            _didIteratorError1 = true;
+                                                                            _iteratorError1 = err;
                                                                         } finally{
-                                                                            if (_didIteratorError1) {
-                                                                                throw _iteratorError1;
+                                                                            try {
+                                                                                if (!_iteratorNormalCompletion1 && _iterator.return != null) {
+                                                                                    _iterator.return();
+                                                                                }
+                                                                            } finally{
+                                                                                if (_didIteratorError1) {
+                                                                                    throw _iteratorError1;
+                                                                                }
                                                                             }
                                                                         }
-                                                                    }
-                                                                    // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
-                                                                    // before this transaction's lock, so a path this process calls "added" can already have
-                                                                    // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
-                                                                    return [
-                                                                        4,
-                                                                        (0, _sharedts.appendRows)(conn, 'preset_files', [
-                                                                            'path',
-                                                                            'preset'
-                                                                        ], 'INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows)
-                                                                    ];
-                                                                case 3:
-                                                                    _state.sent();
-                                                                    return [
-                                                                        2
-                                                                    ];
-                                                            }
-                                                        });
-                                                    })();
-                                                })
-                                            ];
-                                        case 12:
-                                            _state.sent();
-                                            if (!(vanished.length > 0)) return [
-                                                3,
-                                                14
-                                            ];
-                                            return [
-                                                4,
-                                                stages.time('vanished', function() {
-                                                    return conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map(function(p) {
-                                                        return [
-                                                            p
-                                                        ];
-                                                    }));
-                                                })
-                                            ];
-                                        case 13:
-                                            _state.sent();
-                                            _state.label = 14;
-                                        case 14:
-                                            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                                            if (!(touched.length > 0)) return [
-                                                3,
-                                                22
-                                            ];
-                                            _state.label = 15;
-                                        case 15:
-                                            _state.trys.push([
-                                                15,
-                                                20,
-                                                21,
-                                                22
-                                            ]);
-                                            _loop = function() {
-                                                var feature;
-                                                return _ts_generator(this, function(_state) {
-                                                    switch(_state.label){
-                                                        case 0:
-                                                            feature = _step.value;
-                                                            return [
-                                                                4,
-                                                                stages.time((0, _stagests.featureStage)(feature.name, 'remove'), function() {
-                                                                    var _feature_remove;
-                                                                    return (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta);
-                                                                })
-                                                            ];
-                                                        case 1:
-                                                            _state.sent();
-                                                            return [
-                                                                2
-                                                            ];
-                                                    }
-                                                });
-                                            };
-                                            _iterator = features[Symbol.iterator]();
-                                            _state.label = 16;
-                                        case 16:
-                                            if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
-                                                3,
-                                                19
-                                            ];
-                                            return [
-                                                5,
-                                                _ts_values(_loop())
-                                            ];
-                                        case 17:
-                                            _state.sent();
-                                            _state.label = 18;
-                                        case 18:
-                                            _iteratorNormalCompletion = true;
-                                            return [
-                                                3,
-                                                16
-                                            ];
-                                        case 19:
-                                            return [
-                                                3,
-                                                22
-                                            ];
-                                        case 20:
-                                            err = _state.sent();
-                                            _didIteratorError = true;
-                                            _iteratorError = err;
-                                            return [
-                                                3,
-                                                22
-                                            ];
-                                        case 21:
-                                            try {
-                                                if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                                    _iterator.return();
-                                                }
-                                            } finally{
-                                                if (_didIteratorError) {
-                                                    throw _iteratorError;
-                                                }
-                                            }
-                                            return [
-                                                7
-                                            ];
-                                        case 22:
-                                            _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
-                                            _state.label = 23;
-                                        case 23:
-                                            _state.trys.push([
-                                                23,
-                                                28,
-                                                29,
-                                                30
-                                            ]);
-                                            _loop1 = function() {
-                                                var feature, docsForFeature;
-                                                return _ts_generator(this, function(_state) {
-                                                    switch(_state.label){
-                                                        case 0:
-                                                            feature = _step1.value;
-                                                            docsForFeature = parsedDocs.map(function(doc) {
-                                                                return {
-                                                                    path: doc.relPath,
-                                                                    extracted: doc.extracted[feature.name]
-                                                                };
+                                                                        return [
+                                                                            4,
+                                                                            (0, _sharedts.appendRows)(conn, 'preset_files', [
+                                                                                'path',
+                                                                                'preset'
+                                                                            ], 'INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows)
+                                                                        ];
+                                                                    case 3:
+                                                                        _state.sent();
+                                                                        return [
+                                                                            2
+                                                                        ];
+                                                                }
                                                             });
+                                                        })();
+                                                    })
+                                                ];
+                                            case 10:
+                                                _state.sent();
+                                                if (!(vanished.length > 0)) return [
+                                                    3,
+                                                    12
+                                                ];
+                                                return [
+                                                    4,
+                                                    stages.time('vanished', function() {
+                                                        return conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map(function(p) {
                                                             return [
-                                                                4,
-                                                                stages.time((0, _stagests.featureStage)(feature.name, 'store'), function() {
-                                                                    var _feature_store;
-                                                                    return (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta);
-                                                                })
-                                                            ];
-                                                        case 1:
-                                                            _state.sent();
-                                                            return [
-                                                                2
+                                                                p
                                                             ];
+                                                        }));
+                                                    })
+                                                ];
+                                            case 11:
+                                                _state.sent();
+                                                _state.label = 12;
+                                            case 12:
+                                                _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                                if (!(touched.length > 0)) return [
+                                                    3,
+                                                    20
+                                                ];
+                                                _state.label = 13;
+                                            case 13:
+                                                _state.trys.push([
+                                                    13,
+                                                    18,
+                                                    19,
+                                                    20
+                                                ]);
+                                                _loop = function() {
+                                                    var feature;
+                                                    return _ts_generator(this, function(_state) {
+                                                        switch(_state.label){
+                                                            case 0:
+                                                                feature = _step.value;
+                                                                return [
+                                                                    4,
+                                                                    stages.time((0, _stagests.featureStage)(feature.name, 'remove'), function() {
+                                                                        var _feature_remove;
+                                                                        return (_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta);
+                                                                    })
+                                                                ];
+                                                            case 1:
+                                                                _state.sent();
+                                                                return [
+                                                                    2
+                                                                ];
+                                                        }
+                                                    });
+                                                };
+                                                _iterator = features[Symbol.iterator]();
+                                                _state.label = 14;
+                                            case 14:
+                                                if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                                                    3,
+                                                    17
+                                                ];
+                                                return [
+                                                    5,
+                                                    _ts_values(_loop())
+                                                ];
+                                            case 15:
+                                                _state.sent();
+                                                _state.label = 16;
+                                            case 16:
+                                                _iteratorNormalCompletion = true;
+                                                return [
+                                                    3,
+                                                    14
+                                                ];
+                                            case 17:
+                                                return [
+                                                    3,
+                                                    20
+                                                ];
+                                            case 18:
+                                                err = _state.sent();
+                                                _didIteratorError = true;
+                                                _iteratorError = err;
+                                                return [
+                                                    3,
+                                                    20
+                                                ];
+                                            case 19:
+                                                try {
+                                                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                                        _iterator.return();
+                                                    }
+                                                } finally{
+                                                    if (_didIteratorError) {
+                                                        throw _iteratorError;
                                                     }
-                                                });
-                                            };
-                                            _iterator1 = features[Symbol.iterator]();
-                                            _state.label = 24;
-                                        case 24:
-                                            if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
-                                                3,
-                                                27
-                                            ];
-                                            return [
-                                                5,
-                                                _ts_values(_loop1())
-                                            ];
-                                        case 25:
-                                            _state.sent();
-                                            _state.label = 26;
-                                        case 26:
-                                            _iteratorNormalCompletion1 = true;
-                                            return [
-                                                3,
-                                                24
-                                            ];
-                                        case 27:
-                                            return [
-                                                3,
-                                                30
-                                            ];
-                                        case 28:
-                                            err = _state.sent();
-                                            _didIteratorError1 = true;
-                                            _iteratorError1 = err;
-                                            return [
-                                                3,
-                                                30
-                                            ];
-                                        case 29:
-                                            try {
-                                                if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
-                                                    _iterator1.return();
-                                                }
-                                            } finally{
-                                                if (_didIteratorError1) {
-                                                    throw _iteratorError1;
                                                 }
-                                            }
-                                            return [
-                                                7
-                                            ];
-                                        case 30:
-                                            _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
-                                            _state.label = 31;
-                                        case 31:
-                                            _state.trys.push([
-                                                31,
-                                                36,
-                                                37,
-                                                38
-                                            ]);
-                                            _loop2 = function() {
-                                                var feature;
-                                                return _ts_generator(this, function(_state) {
-                                                    switch(_state.label){
-                                                        case 0:
-                                                            feature = _step2.value;
-                                                            return [
-                                                                4,
-                                                                stages.time((0, _stagests.featureStage)(feature.name, 'after'), function() {
-                                                                    var _feature_afterReconcile;
-                                                                    return (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta);
-                                                                })
-                                                            ];
-                                                        case 1:
-                                                            _state.sent();
-                                                            return [
-                                                                2
-                                                            ];
+                                                return [
+                                                    7
+                                                ];
+                                            case 20:
+                                                _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                                                _state.label = 21;
+                                            case 21:
+                                                _state.trys.push([
+                                                    21,
+                                                    26,
+                                                    27,
+                                                    28
+                                                ]);
+                                                _loop1 = function() {
+                                                    var feature, docsForFeature;
+                                                    return _ts_generator(this, function(_state) {
+                                                        switch(_state.label){
+                                                            case 0:
+                                                                feature = _step1.value;
+                                                                docsForFeature = parsedDocs.map(function(doc) {
+                                                                    return {
+                                                                        path: doc.relPath,
+                                                                        extracted: doc.extracted[feature.name]
+                                                                    };
+                                                                });
+                                                                return [
+                                                                    4,
+                                                                    stages.time((0, _stagests.featureStage)(feature.name, 'store'), function() {
+                                                                        var _feature_store;
+                                                                        return (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta);
+                                                                    })
+                                                                ];
+                                                            case 1:
+                                                                _state.sent();
+                                                                return [
+                                                                    2
+                                                                ];
+                                                        }
+                                                    });
+                                                };
+                                                _iterator1 = features[Symbol.iterator]();
+                                                _state.label = 22;
+                                            case 22:
+                                                if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
+                                                    3,
+                                                    25
+                                                ];
+                                                return [
+                                                    5,
+                                                    _ts_values(_loop1())
+                                                ];
+                                            case 23:
+                                                _state.sent();
+                                                _state.label = 24;
+                                            case 24:
+                                                _iteratorNormalCompletion1 = true;
+                                                return [
+                                                    3,
+                                                    22
+                                                ];
+                                            case 25:
+                                                return [
+                                                    3,
+                                                    28
+                                                ];
+                                            case 26:
+                                                err = _state.sent();
+                                                _didIteratorError1 = true;
+                                                _iteratorError1 = err;
+                                                return [
+                                                    3,
+                                                    28
+                                                ];
+                                            case 27:
+                                                try {
+                                                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                                                        _iterator1.return();
+                                                    }
+                                                } finally{
+                                                    if (_didIteratorError1) {
+                                                        throw _iteratorError1;
                                                     }
-                                                });
-                                            };
-                                            _iterator2 = features[Symbol.iterator]();
-                                            _state.label = 32;
-                                        case 32:
-                                            if (!!(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done)) return [
-                                                3,
-                                                35
-                                            ];
-                                            return [
-                                                5,
-                                                _ts_values(_loop2())
-                                            ];
-                                        case 33:
-                                            _state.sent();
-                                            _state.label = 34;
-                                        case 34:
-                                            _iteratorNormalCompletion2 = true;
-                                            return [
-                                                3,
-                                                32
-                                            ];
-                                        case 35:
-                                            return [
-                                                3,
-                                                38
-                                            ];
-                                        case 36:
-                                            err = _state.sent();
-                                            _didIteratorError2 = true;
-                                            _iteratorError2 = err;
-                                            return [
-                                                3,
-                                                38
-                                            ];
-                                        case 37:
-                                            try {
-                                                if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
-                                                    _iterator2.return();
                                                 }
-                                            } finally{
-                                                if (_didIteratorError2) {
-                                                    throw _iteratorError2;
+                                                return [
+                                                    7
+                                                ];
+                                            case 28:
+                                                _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+                                                _state.label = 29;
+                                            case 29:
+                                                _state.trys.push([
+                                                    29,
+                                                    34,
+                                                    35,
+                                                    36
+                                                ]);
+                                                _loop2 = function() {
+                                                    var feature;
+                                                    return _ts_generator(this, function(_state) {
+                                                        switch(_state.label){
+                                                            case 0:
+                                                                feature = _step2.value;
+                                                                return [
+                                                                    4,
+                                                                    stages.time((0, _stagests.featureStage)(feature.name, 'after'), function() {
+                                                                        var _feature_afterReconcile;
+                                                                        return (_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta);
+                                                                    })
+                                                                ];
+                                                            case 1:
+                                                                _state.sent();
+                                                                return [
+                                                                    2
+                                                                ];
+                                                        }
+                                                    });
+                                                };
+                                                _iterator2 = features[Symbol.iterator]();
+                                                _state.label = 30;
+                                            case 30:
+                                                if (!!(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done)) return [
+                                                    3,
+                                                    33
+                                                ];
+                                                return [
+                                                    5,
+                                                    _ts_values(_loop2())
+                                                ];
+                                            case 31:
+                                                _state.sent();
+                                                _state.label = 32;
+                                            case 32:
+                                                _iteratorNormalCompletion2 = true;
+                                                return [
+                                                    3,
+                                                    30
+                                                ];
+                                            case 33:
+                                                return [
+                                                    3,
+                                                    36
+                                                ];
+                                            case 34:
+                                                err = _state.sent();
+                                                _didIteratorError2 = true;
+                                                _iteratorError2 = err;
+                                                return [
+                                                    3,
+                                                    36
+                                                ];
+                                            case 35:
+                                                try {
+                                                    if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
+                                                        _iterator2.return();
+                                                    }
+                                                } finally{
+                                                    if (_didIteratorError2) {
+                                                        throw _iteratorError2;
+                                                    }
                                                 }
-                                            }
-                                            return [
-                                                7
-                                            ];
-                                        case 38:
-                                            return [
-                                                2
-                                            ];
-                                    }
-                                });
-                            })();
+                                                return [
+                                                    7
+                                                ];
+                                            case 36:
+                                                return [
+                                                    2
+                                                ];
+                                        }
+                                    });
+                                })();
+                            },
+                            finish: function finish(txMs) {
+                                return stages.take(elapsed(), txMs, timing.workerParseMs);
+                            },
+                            record: function record(conn, txMs) {
+                                return _async_to_generator(function() {
+                                    return _ts_generator(this, function(_state) {
+                                        switch(_state.label){
+                                            case 0:
+                                                (0, _lockwaitts.recordLockWaitMs)(configDir, txMs);
+                                                if (!dialect.recordDuration) return [
+                                                    3,
+                                                    2
+                                                ];
+                                                return [
+                                                    4,
+                                                    stages.time('meta', function() {
+                                                        var _dialect_recordDuration;
+                                                        return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, txMs);
+                                                    })
+                                                ];
+                                            case 1:
+                                                _state.sent();
+                                                _state.label = 2;
+                                            case 2:
+                                                return [
+                                                    2
+                                                ];
+                                        }
+                                    });
+                                })();
+                            }
+                        }
+                    ];
+            }
+        });
+    })();
+}
+function reconcile(conn, cfg, paths, dialect, pool, forcedPaths) {
+    return _async_to_generator(function() {
+        var timing, snapshot, plan, txStart, txMs;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    timing = createReconcileTiming(cfg);
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return readReconcileSnapshot(conn, timing);
+                        })
+                    ];
+                case 1:
+                    snapshot = _state.sent();
+                    return [
+                        4,
+                        prepareReconcile(snapshot, cfg, paths, dialect, timing, pool, forcedPaths)
+                    ];
+                case 2:
+                    plan = _state.sent();
+                    if (!plan.mutated) return [
+                        2,
+                        {
+                            parsed: plan.parsed,
+                            warnings: plan.warnings,
+                            stages: plan.finish(0)
+                        }
+                    ];
+                    txStart = Date.now();
+                    _state.label = 3;
+                case 3:
+                    _state.trys.push([
+                        3,
+                        ,
+                        5,
+                        6
+                    ]);
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return plan.apply(conn);
                         }, dialect.beginMode())
                     ];
-                case 5:
+                case 4:
                     _state.sent();
-                    durationMs = Date.now() - txStart;
-                    // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.
-                    (0, _lockwaitts.recordLockWaitMs)(configDir, durationMs);
-                    if (!dialect.recordDuration) return [
+                    return [
                         3,
+                        6
+                    ];
+                case 5:
+                    txMs = Date.now() - txStart;
+                    timing.txMs += txMs;
+                    return [
                         7
                     ];
+                case 6:
                     return [
                         4,
-                        stages.time('meta', function() {
-                            var _dialect_recordDuration;
-                            return (_dialect_recordDuration = dialect.recordDuration) === null || _dialect_recordDuration === void 0 ? void 0 : _dialect_recordDuration.call(dialect, conn, durationMs);
-                        })
+                        plan.record(conn, txMs)
                     ];
-                case 6:
-                    _state.sent();
-                    _state.label = 7;
                 case 7:
+                    _state.sent();
                     return [
                         2,
                         {
-                            parsed: parsedDocs.length,
-                            warnings: warnings,
-                            stages: stages.take(elapsed(), durationMs, workerParseMs)
+                            parsed: plan.parsed,
+                            warnings: plan.warnings,
+                            stages: plan.finish(timing.txMs)
                         }
                     ];
             }