sensemaking 0.25.1 → 0.27.0

package/dist/cjs/store/duckdb/reconcile.js

@@ -11,6 +11,15 @@ Object.defineProperty(exports, "duckdbDialect", {
 var _errorsts = require("../../errors.js");
 var _sharedts = require("../shared.js");
 var _lexicalts = require("./lexical.js");
+var _nativets = require("./native.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -37,6 +46,23 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _instanceof(left, right) {
+    "@swc/helpers - instanceof";
+    if (right != null && typeof Symbol !== "undefined" && right[Symbol.hasInstance]) {
+        return !!right[Symbol.hasInstance](left);
+    } else return left instanceof right;
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
 function _ts_generator(thisArg, body) {
     var f, y, t, _ = {
         label: 0,
@@ -136,9 +162,41 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
+function _ts_values(o) {
+    var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
+    if (m) {
+        return m.call(o);
+    }
+    if (o && typeof o.length === "number") {
+        return {
+            next: function() {
+                if (o && i >= o.length) {
+                    o = void 0;
+                }
+                return {
+                    value: o && o[i++],
+                    done: !o
+                };
+            }
+        };
+    }
+    throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
+}
+function _type_of(obj) {
+    "@swc/helpers - typeof";
+    return obj && typeof Symbol !== "undefined" && obj.constructor === Symbol ? "symbol" : typeof obj;
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
-// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here
-// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
+// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so changed rows are maintained
+// here; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
 // column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 // No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):
 // `path` is content's own primary key, so this is a plain per-doc row.
@@ -149,6 +207,8 @@ var CONTENT_COLUMNS = [
     'text'
 ];
 var INSERT_CONTENT_SQL = "INSERT INTO content (".concat(CONTENT_COLUMNS.map(_sharedts.quoteIdent).join(', '), ") VALUES (?, ?, ?, ?)");
+// Bound the extra persisted text held for exact comparison, even during a full reparse.
+var CONTENT_LOOKUP_CHUNK = 128;
 function contentRow(doc) {
     return [
         doc.relPath,
@@ -160,43 +220,500 @@ function contentRow(doc) {
 // No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway
 // so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.
 var MAX_FRONTMATTER_COLUMNS = 10000;
-function reconcileContent(conn, touched, docs, _delta) {
+var FRONTMATTER_UPDATE_CHUNK = 128;
+function withNativeStatement(native, sql, fn) {
     return _async_to_generator(function() {
+        var stmt, errors, result, err;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    if (!(touched.length > 0)) return [
+                    return [
+                        4,
+                        native.prepare(sql)
+                    ];
+                case 1:
+                    stmt = _state.sent();
+                    errors = [];
+                    _state.label = 2;
+                case 2:
+                    _state.trys.push([
+                        2,
+                        4,
+                        ,
+                        5
+                    ]);
+                    return [
+                        4,
+                        fn(stmt)
+                    ];
+                case 3:
+                    result = _state.sent();
+                    return [
+                        3,
+                        5
+                    ];
+                case 4:
+                    err = _state.sent();
+                    errors.push(err);
+                    return [
+                        3,
+                        5
+                    ];
+                case 5:
+                    try {
+                        stmt.destroySync();
+                    } catch (err) {
+                        errors.push(err);
+                    }
+                    if (errors.length === 1) throw errors[0];
+                    if (errors.length > 1) throw new AggregateError(errors, 'frontmatter statement and cleanup failed');
+                    return [
+                        2,
+                        result
+                    ];
+            }
+        });
+    })();
+}
+// Wide writes amplify native commit allocations even for one changed stamp.
+// Skip only known, exactly equal values; uncertain representations still write.
+function updateFrontmatter(conn, writableColumns, rows) {
+    return _async_to_generator(function() {
+        var _loop, native, _ref, DuckDBVariantValue, DuckDBTypeId, physical, writable, pathIndex, columnTypes, insertSql, sameValue, i;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _loop = function(i) {
+                        var _loop, chunk, paths, placeholders, selected, existing, plans, position;
+                        return _ts_generator(this, function(_state) {
+                            switch(_state.label){
+                                case 0:
+                                    _loop = function(position1) {
+                                        var plan, end, assignments;
+                                        return _ts_generator(this, function(_state) {
+                                            switch(_state.label){
+                                                case 0:
+                                                    plan = plans[position1];
+                                                    if (!plan.missing && plan.changed.length === 0) {
+                                                        ++position1;
+                                                        return [
+                                                            2,
+                                                            (position = position1, "continue")
+                                                        ];
+                                                    }
+                                                    end = position1 + 1;
+                                                    while(end < plans.length && plans[end].missing === plan.missing && plans[end].signature === plan.signature)++end;
+                                                    if (!plan.missing) return [
+                                                        3,
+                                                        2
+                                                    ];
+                                                    return [
+                                                        4,
+                                                        (0, _sharedts.appendRows)(conn, 'frontmatter', writableColumns, insertSql, plans.slice(position1, end).map(function(repair) {
+                                                            return repair.row;
+                                                        }))
+                                                    ];
+                                                case 1:
+                                                    _state.sent();
+                                                    return [
+                                                        3,
+                                                        4
+                                                    ];
+                                                case 2:
+                                                    assignments = plan.changed.map(function(index) {
+                                                        return "".concat((0, _sharedts.quoteIdent)(writableColumns[index]), " = ?");
+                                                    }).join(', ');
+                                                    return [
+                                                        4,
+                                                        withNativeStatement(native, "UPDATE frontmatter SET ".concat(assignments, ' WHERE "path" = ?'), function(stmt) {
+                                                            return _async_to_generator(function() {
+                                                                var _loop, rowIndex;
+                                                                return _ts_generator(this, function(_state) {
+                                                                    switch(_state.label){
+                                                                        case 0:
+                                                                            _loop = function(rowIndex) {
+                                                                                var row;
+                                                                                return _ts_generator(this, function(_state) {
+                                                                                    switch(_state.label){
+                                                                                        case 0:
+                                                                                            row = plans[rowIndex].row;
+                                                                                            stmt.bind(_to_consumable_array(plan.changed.map(function(index) {
+                                                                                                var _row_index;
+                                                                                                return (_row_index = row[index]) !== null && _row_index !== void 0 ? _row_index : null;
+                                                                                            })).concat([
+                                                                                                row[pathIndex]
+                                                                                            ]));
+                                                                                            return [
+                                                                                                4,
+                                                                                                stmt.run()
+                                                                                            ];
+                                                                                        case 1:
+                                                                                            _state.sent();
+                                                                                            return [
+                                                                                                2
+                                                                                            ];
+                                                                                    }
+                                                                                });
+                                                                            };
+                                                                            rowIndex = position1;
+                                                                            _state.label = 1;
+                                                                        case 1:
+                                                                            if (!(rowIndex < end)) return [
+                                                                                3,
+                                                                                4
+                                                                            ];
+                                                                            return [
+                                                                                5,
+                                                                                _ts_values(_loop(rowIndex))
+                                                                            ];
+                                                                        case 2:
+                                                                            _state.sent();
+                                                                            _state.label = 3;
+                                                                        case 3:
+                                                                            ++rowIndex;
+                                                                            return [
+                                                                                3,
+                                                                                1
+                                                                            ];
+                                                                        case 4:
+                                                                            return [
+                                                                                2
+                                                                            ];
+                                                                    }
+                                                                });
+                                                            })();
+                                                        })
+                                                    ];
+                                                case 3:
+                                                    _state.sent();
+                                                    _state.label = 4;
+                                                case 4:
+                                                    position1 = end;
+                                                    position = position1, void 0;
+                                                    return [
+                                                        2
+                                                    ];
+                                            }
+                                        });
+                                    };
+                                    chunk = rows.slice(i, i + FRONTMATTER_UPDATE_CHUNK);
+                                    paths = chunk.map(function(row) {
+                                        return row[pathIndex];
+                                    });
+                                    placeholders = paths.map(function() {
+                                        return '?';
+                                    }).join(', ');
+                                    selected = writableColumns.map(_sharedts.quoteIdent).join(', ');
+                                    return [
+                                        4,
+                                        withNativeStatement(native, "SELECT ".concat(selected, ' FROM frontmatter WHERE "path" IN (').concat(placeholders, ")"), function(stmt) {
+                                            return _async_to_generator(function() {
+                                                var reader;
+                                                return _ts_generator(this, function(_state) {
+                                                    switch(_state.label){
+                                                        case 0:
+                                                            stmt.bind(paths);
+                                                            return [
+                                                                4,
+                                                                stmt.runAndReadAll()
+                                                            ];
+                                                        case 1:
+                                                            reader = _state.sent();
+                                                            // Native wrappers distinguish numeric tags and SQL NULL from VARIANT null.
+                                                            return [
+                                                                2,
+                                                                new Map(reader.getRowObjects().map(function(row) {
+                                                                    return [
+                                                                        row.path,
+                                                                        row
+                                                                    ];
+                                                                }))
+                                                            ];
+                                                    }
+                                                });
+                                            })();
+                                        })
+                                    ];
+                                case 1:
+                                    existing = _state.sent();
+                                    plans = chunk.map(function(row) {
+                                        var old = existing.get(row[pathIndex]);
+                                        var changed = old ? writableColumns.flatMap(function(column, index) {
+                                            return sameValue(old[column], row[index], column) ? [] : [
+                                                index
+                                            ];
+                                        }) : [];
+                                        return {
+                                            row: row,
+                                            missing: old === undefined,
+                                            changed: changed,
+                                            signature: changed.join(',')
+                                        };
+                                    });
+                                    position = 0;
+                                    _state.label = 2;
+                                case 2:
+                                    if (!(position < plans.length)) return [
+                                        3,
+                                        5
+                                    ];
+                                    return [
+                                        5,
+                                        _ts_values(_loop(position))
+                                    ];
+                                case 3:
+                                    _state.sent();
+                                    _state.label = 4;
+                                case 4:
+                                    return [
+                                        3,
+                                        2
+                                    ];
+                                case 5:
+                                    return [
+                                        2
+                                    ];
+                            }
+                        });
+                    };
+                    if (rows.length === 0) return [
+                        2
+                    ];
+                    native = conn.duckdb;
+                    return [
+                        4,
+                        (0, _nativets.duckdbApi)()
+                    ];
+                case 1:
+                    _ref = _state.sent(), DuckDBVariantValue = _ref.DuckDBVariantValue, DuckDBTypeId = _ref.DuckDBTypeId;
+                    return [
+                        4,
+                        withNativeStatement(native, 'PRAGMA table_info(frontmatter)', function(stmt) {
+                            return _async_to_generator(function() {
+                                var reader;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                stmt.runAndReadAll()
+                                            ];
+                                        case 1:
+                                            reader = _state.sent();
+                                            return [
+                                                2,
+                                                reader.getRowObjectsJS()
+                                            ];
+                                    }
+                                });
+                            })();
+                        })
+                    ];
+                case 2:
+                    physical = _state.sent();
+                    writable = new Map(writableColumns.map(function(name, i) {
+                        return [
+                            name,
+                            i
+                        ];
+                    }));
+                    pathIndex = writable.get('path');
+                    if (pathIndex === undefined) throw new Error('frontmatter update requires a writable path column');
+                    columnTypes = new Map(physical.map(function(column) {
+                        return [
+                            column.name,
+                            column.type
+                        ];
+                    }));
+                    insertSql = "INSERT INTO frontmatter (".concat(writableColumns.map(_sharedts.quoteIdent).join(', '), ") VALUES (").concat(writableColumns.map(function() {
+                        return '?';
+                    }).join(', '), ")");
+                    sameValue = function sameValue(old, value, column) {
+                        var _old_type;
+                        if (value === null || value === undefined) return old === null;
+                        if (columnTypes.get(column) !== 'VARIANT') return Object.is(old, value);
+                        if (!_instanceof(old, DuckDBVariantValue)) return false;
+                        var typeId = typeof value === 'string' ? DuckDBTypeId.VARCHAR : (typeof value === "undefined" ? "undefined" : _type_of(value)) === 'bigint' ? DuckDBTypeId.HUGEINT : typeof value === 'number' && !Number.isInteger(value) ? DuckDBTypeId.DOUBLE : undefined;
+                        return typeId !== undefined && ((_old_type = old.type) === null || _old_type === void 0 ? void 0 : _old_type.typeId) === typeId && Object.is(old.value, value);
+                    };
+                    i = 0;
+                    _state.label = 3;
+                case 3:
+                    if (!(i < rows.length)) return [
                         3,
+                        6
+                    ];
+                    return [
+                        5,
+                        _ts_values(_loop(i))
+                    ];
+                case 4:
+                    _state.sent();
+                    _state.label = 5;
+                case 5:
+                    i += FRONTMATTER_UPDATE_CHUNK;
+                    return [
+                        3,
+                        3
+                    ];
+                case 6:
+                    return [
                         2
                     ];
+            }
+        });
+    })();
+}
+function reconcileContent(conn, touched, docs, delta) {
+    return _async_to_generator(function() {
+        var added, reparsed, changed, removed, i, _stmt, chunk, stmt, rows, existing, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, doc, row, docPaths, vanished, i1, _stmt1, _removed, chunk1, stmt1, rows1, changedDocs;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    added = new Set(delta.added);
+                    reparsed = docs.filter(function(doc) {
+                        return !added.has(doc.relPath);
+                    });
+                    changed = new Set(delta.added);
+                    removed = [];
+                    i = 0;
+                    _state.label = 1;
+                case 1:
+                    if (!(i < reparsed.length)) return [
+                        3,
+                        5
+                    ];
+                    chunk = reparsed.slice(i, i + CONTENT_LOOKUP_CHUNK);
                     return [
                         4,
-                        conn.runBatch('DELETE FROM content WHERE "path" = ?', touched.map(function(p) {
+                        conn.prepare('SELECT "path", title, summary, text FROM content WHERE "path" IN ('.concat(chunk.map(function() {
+                            return '?';
+                        }).join(', '), ")"))
+                    ];
+                case 2:
+                    stmt = _state.sent();
+                    return [
+                        4,
+                        (_stmt = stmt).all.apply(_stmt, _to_consumable_array(chunk.map(function(doc) {
+                            return doc.relPath;
+                        })))
+                    ];
+                case 3:
+                    rows = _state.sent();
+                    existing = new Map(rows.map(function(row) {
+                        return [
+                            row.path,
+                            row
+                        ];
+                    }));
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = chunk[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            doc = _step.value;
+                            row = existing.get(doc.relPath);
+                            if (row && row.title === doc.search.title && row.summary === doc.search.summary && row.text === doc.search.text) continue;
+                            changed.add(doc.relPath);
+                            if (row) removed.push(doc.relPath);
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
+                    _state.label = 4;
+                case 4:
+                    i += CONTENT_LOOKUP_CHUNK;
+                    return [
+                        3,
+                        1
+                    ];
+                case 5:
+                    docPaths = new Set(docs.map(function(doc) {
+                        return doc.relPath;
+                    }));
+                    vanished = touched.filter(function(path) {
+                        return !docPaths.has(path);
+                    });
+                    i1 = 0;
+                    _state.label = 6;
+                case 6:
+                    if (!(i1 < vanished.length)) return [
+                        3,
+                        10
+                    ];
+                    chunk1 = vanished.slice(i1, i1 + CONTENT_LOOKUP_CHUNK);
+                    return [
+                        4,
+                        conn.prepare('SELECT "path" FROM content WHERE "path" IN ('.concat(chunk1.map(function() {
+                            return '?';
+                        }).join(', '), ")"))
+                    ];
+                case 7:
+                    stmt1 = _state.sent();
+                    return [
+                        4,
+                        (_stmt1 = stmt1).all.apply(_stmt1, _to_consumable_array(chunk1))
+                    ];
+                case 8:
+                    rows1 = _state.sent();
+                    (_removed = removed).push.apply(_removed, _to_consumable_array(rows1.map(function(row) {
+                        return row.path;
+                    })));
+                    _state.label = 9;
+                case 9:
+                    i1 += CONTENT_LOOKUP_CHUNK;
+                    return [
+                        3,
+                        6
+                    ];
+                case 10:
+                    if (!(removed.length > 0)) return [
+                        3,
+                        12
+                    ];
+                    return [
+                        4,
+                        conn.runBatch('DELETE FROM content WHERE "path" = ?', removed.map(function(p) {
                             return [
                                 p
                             ];
                         }))
                     ];
-                case 1:
+                case 11:
                     _state.sent();
-                    _state.label = 2;
-                case 2:
-                    // The delete above cleared every touched path, and duckdb holds the cache file for its whole
-                    // connection, so no second writer can have landed one of these rows: nothing can conflict.
+                    _state.label = 12;
+                case 12:
+                    // Keep document order and the existing insert/error semantics for added paths.
+                    changedDocs = docs.filter(function(doc) {
+                        return changed.has(doc.relPath);
+                    });
                     return [
                         4,
-                        (0, _sharedts.appendRows)(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow))
+                        (0, _sharedts.appendRows)(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, changedDocs.map(contentRow))
                     ];
-                case 3:
+                case 13:
                     _state.sent();
-                    // Build prepares the FTS index after reconcile. This persisted mark commits with the content
-                    // change, so an interrupted build cannot expose stale lexical results.
+                    if (!(removed.length > 0 || changedDocs.length > 0)) return [
+                        3,
+                        15
+                    ];
                     return [
                         4,
                         (0, _lexicalts.markContentStale)(conn)
                     ];
-                case 4:
+                case 14:
                     _state.sent();
+                    _state.label = 15;
+                case 15:
                     return [
                         2
                     ];
@@ -242,6 +759,7 @@ var duckdbDialect = {
         }
     },
     addColumns: addColumns,
+    updateFrontmatter: updateFrontmatter,
     reconcileContent: reconcileContent
 };
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }